Move roxie row allocator into roxiehelper

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/roxiehelper/CMakeLists.txt

@@ -44,7 +44,8 @@ include_directories (            ./../../system/security/securesocket
 
 ADD_DEFINITIONS( -DROXIEHELPER_EXPORTS -D_USRDLL -DMODULE_PRIORITY=5 )
 
-set ( SRCS roxiehelper.cpp roxiedebug.cpp roxiehelper.hpp roxiedebug.hpp roxielmj.hpp)
+set ( SRCS roxiehelper.cpp roxiedebug.cpp roxierow.cpp
+           roxiehelper.hpp roxiedebug.hpp roxierow.hpp roxielmj.hpp)
 
 HPCC_ADD_LIBRARY( roxiehelper SHARED ${SRCS} )
 install ( TARGETS roxiehelper DESTINATION ${OSSDIR}/lib )

common/roxiehelper/roxierow.cpp

@@ -0,0 +1,248 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jexcept.hpp"
+#include "roxierow.hpp"
+#include "thorcommon.ipp"
+
+class RoxieEngineRowAllocatorBase : public CInterface, implements IEngineRowAllocator
+{
+public:
+    RoxieEngineRowAllocatorBase(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId)
+        : rowManager(_rowManager), meta(_meta)
+    {
+        activityId = _activityId;
+        allocatorId = _allocatorId;
+    }
+
+    IMPLEMENT_IINTERFACE
+
+//interface IEngineRowsetAllocator
+    virtual byte * * createRowset(unsigned count)
+    {
+        if (count == 0)
+            return NULL;
+        return (byte **) rowManager.allocate(count * sizeof(void *), allocatorId | ACTIVITY_FLAG_ISREGISTERED);
+    }
+
+    virtual void releaseRowset(unsigned count, byte * * rowset)
+    {
+        rtlReleaseRowset(count, rowset);
+    }
+
+    virtual byte * * linkRowset(byte * * rowset)
+    {
+        return rtlLinkRowset(rowset);
+    }
+
+    virtual byte * * appendRowOwn(byte * * rowset, unsigned newRowCount, void * row)
+    {
+        byte * * expanded = doReallocRows(rowset, newRowCount-1, newRowCount);
+        expanded[newRowCount-1] = (byte *)row;
+        return expanded;
+    }
+
+    virtual byte * * reallocRows(byte * * rowset, unsigned oldRowCount, unsigned newRowCount)
+    {
+        //New rows (if any) aren't cleared....
+        return doReallocRows(rowset, oldRowCount, newRowCount);
+    }
+
+    inline byte * * doReallocRows(byte * * rowset, unsigned oldRowCount, unsigned newRowCount)
+    {
+        if (!rowset)
+            return createRowset(newRowCount);
+
+        //This would be more efficient if previous capacity was stored by the caller - or if capacity() is more efficient
+        if (newRowCount * sizeof(void *) <= RoxieRowCapacity(rowset))
+            return rowset;
+
+        size32_t capacity;
+        return (byte * *)rowManager.resizeRow(rowset, oldRowCount * sizeof(void *), newRowCount * sizeof(void *), allocatorId | ACTIVITY_FLAG_ISREGISTERED, capacity);
+    }
+
+    virtual void releaseRow(const void * row)
+    {
+        ReleaseRoxieRow(row);
+    }
+
+    virtual void * linkRow(const void * row)
+    {
+        LinkRoxieRow(row);
+        return const_cast<void *>(row);
+    }
+
+    virtual IOutputMetaData * queryOutputMeta()
+    {
+        return meta.queryOriginal();
+    }
+    virtual unsigned queryActivityId()
+    {
+        return activityId;
+    }
+    virtual StringBuffer &getId(StringBuffer &idStr)
+    {
+        return idStr.append(activityId); // MORE - may want more context info in here
+    }
+    virtual IOutputRowSerializer *createRowSerializer(ICodeContext *ctx)
+    {
+        return meta.createRowSerializer(ctx, activityId);
+    }
+    virtual IOutputRowDeserializer *createRowDeserializer(ICodeContext *ctx)
+    {
+        return meta.createRowDeserializer(ctx, activityId);
+    }
+protected:
+    roxiemem::IRowManager & rowManager;
+    const CachedOutputMetaData meta;
+    unsigned activityId;
+    unsigned allocatorId;
+};
+
+//General purpose row allocator here for reference - should be removed once spcialised versions are created
+class RoxieEngineRowAllocator : public RoxieEngineRowAllocatorBase
+{
+public:
+    RoxieEngineRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId)
+        : RoxieEngineRowAllocatorBase(_rowManager, _meta, _activityId, _allocatorId)
+    {
+    }
+
+    virtual void * createRow()
+    {
+        size32_t allocSize = meta.getInitialSize();
+        return rowManager.allocate(allocSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED);
+    }
+
+    virtual void * createRow(size32_t & allocatedSize)
+    {
+        const size32_t allocSize = meta.getInitialSize();
+        void *ret = rowManager.allocate(allocSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED);
+        //more: allocate could return the allocated size, but that would penalise the fixed row case
+        allocatedSize = RoxieRowCapacity(ret);
+        return ret;
+    }
+
+    virtual void * resizeRow(size32_t newSize, void * row, size32_t & size)
+    {
+        return rowManager.resizeRow(row, size, newSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED, size);
+    }
+
+    virtual void * finalizeRow(size32_t finalSize, void * row, size32_t oldSize)
+    {
+        unsigned id = allocatorId | ACTIVITY_FLAG_ISREGISTERED;
+        if (meta.needsDestruct()) id |= ACTIVITY_FLAG_NEEDSDESTRUCTOR;
+        return rowManager.finalizeRow(row, oldSize, finalSize, id);
+    }
+};
+
+class RoxieEngineFixedRowAllocator : public RoxieEngineRowAllocatorBase
+{
+public:
+    RoxieEngineFixedRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId, bool packed)
+        : RoxieEngineRowAllocatorBase(_rowManager, _meta, _activityId, _allocatorId)
+    {
+        unsigned flags = packed ? roxiemem::RHFpacked : roxiemem::RHFnone;
+        if (meta.needsDestruct())
+            flags |= roxiemem::RHFhasdestructor;
+        heap.setown(rowManager.createFixedRowHeap(meta.getFixedSize(), allocatorId | ACTIVITY_FLAG_ISREGISTERED, (roxiemem::RoxieHeapFlags)flags));
+    }
+
+    virtual void * createRow()
+    {
+        return heap->allocate();
+    }
+
+    virtual void * createRow(size32_t & allocatedSize)
+    {
+        allocatedSize = meta.getInitialSize();
+        return heap->allocate();
+    }
+
+    virtual void * resizeRow(size32_t newSize, void * row, size32_t & size)
+    {
+        throwUnexpected();
+        return NULL;
+    }
+
+    virtual void * finalizeRow(size32_t finalSize, void * row, size32_t oldSize)
+    {
+        if (!meta.needsDestruct())
+            return row;
+        return heap->finalizeRow(row);
+    }
+
+protected:
+    Owned<roxiemem::IFixedRowHeap> heap;
+};
+
+class RoxieEngineVariableRowAllocator : public RoxieEngineRowAllocatorBase
+{
+public:
+    RoxieEngineVariableRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId, bool _packed)
+        : RoxieEngineRowAllocatorBase(_rowManager, _meta, _activityId, _allocatorId), packed(_packed)
+    {
+        unsigned flags = packed ? roxiemem::RHFpacked : roxiemem::RHFnone;
+        if (meta.needsDestruct())
+            flags |= roxiemem::RHFhasdestructor;
+        heap.setown(rowManager.createVariableRowHeap(allocatorId | ACTIVITY_FLAG_ISREGISTERED, (roxiemem::RoxieHeapFlags)flags));
+    }
+
+    virtual void * createRow()
+    {
+        size32_t allocSize = meta.getInitialSize();
+        size32_t capacity;
+        return heap->allocate(allocSize, capacity);
+    }
+
+    virtual void * createRow(size32_t & allocatedSize)
+    {
+        const size32_t allocSize = meta.getInitialSize();
+        return heap->allocate(allocSize, allocatedSize);
+    }
+
+    virtual void * resizeRow(size32_t newSize, void * row, size32_t & size)
+    {
+        return heap->resizeRow(row, size, newSize, size);
+    }
+
+    virtual void * finalizeRow(size32_t finalSize, void * row, size32_t oldSize)
+    {
+        if (!meta.needsDestruct() && !packed)
+            return row;
+        return heap->finalizeRow(row, oldSize, finalSize);
+    }
+
+protected:
+    Owned<roxiemem::IVariableRowHeap> heap;
+    bool packed;    // may not be needed - depends on implementation
+};
+
+
+IEngineRowAllocator * createRoxieRowAllocator(roxiemem::IRowManager & rowManager, IOutputMetaData * meta, unsigned activityId, unsigned allocatorId, bool packed)
+{
+#if 0
+    //old code
+    return new RoxieEngineRowAllocator(rowManager, meta, activityId, allocatorId);
+#else
+    if (meta->getFixedSize() != 0)
+        return new RoxieEngineFixedRowAllocator(rowManager, meta, activityId, allocatorId, packed);
+    else
+        return new RoxieEngineVariableRowAllocator(rowManager, meta, activityId, allocatorId, packed);
+#endif
+}

common/roxiehelper/roxierow.hpp

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _ROXIEROW_INCL
+#define _ROXIEROW_INCL
+
+#ifdef _WIN32
+ #ifdef ROXIEHELPER_EXPORTS
+  #define ROXIEHELPER_API __declspec(dllexport)
+ #else
+  #define ROXIEHELPER_API __declspec(dllimport)
+ #endif
+#else
+ #define ROXIEHELPER_API
+#endif
+
+#include "roxiemem.hpp"
+#include "eclhelper.hpp"
+
+extern ROXIEHELPER_API IEngineRowAllocator * createRoxieRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId, bool packed);
+
+#endif

ecl/hthor/hthor.cpp

@@ -26,6 +26,7 @@
 #include "jlzw.hpp"
 #include "jisem.hpp"
 #include "roxiedebug.hpp"
+#include "roxierow.hpp"
 #include "eclhelper.hpp"
 #include "workunit.hpp"
 #include "jfile.hpp"
@@ -9720,7 +9721,7 @@ IHThorException * makeHThorException(ThorActivityKind kind, unsigned activityId,
 
 extern HTHOR_API IEngineRowAllocator * createHThorRowAllocator(IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId)
 {
-    return new HThorEngineRowAllocator(_rowManager, _meta, _activityId, _allocatorId);  
+    return createRoxieRowAllocator(_rowManager, _meta, _activityId, _allocatorId, false);
 }
 
 static class RowCallbackHook : implements IRtlRowCallback

ecl/hthor/hthor.ipp

@@ -54,131 +54,6 @@ typedef roxiemem::OwnedRoxieRow OwnedRow;
 byte * * linkHThorRowset(byte * * rowset);
 void releaseHThorRowset(unsigned count, byte * * rowset);
 
-
-//HThorEngineRowAllocator Copied from RoxieEngineRowAllocator in ccdquery.hpp
-class HThorEngineRowAllocator : public CInterface, implements IEngineRowAllocator
-{
-public:
-    HThorEngineRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId)
-        : rowManager(_rowManager), meta(_meta) 
-    {
-        activityId = _activityId;
-        allocatorId = _allocatorId;
-    }
-    IMPLEMENT_IINTERFACE
-
-//interface IEngineRowsetAllocator
-
-    virtual byte * * createRowset(unsigned count)
-    {
-        if (count == 0)
-            return NULL;
-        return (byte **) rowManager.allocate(count * sizeof(void *), allocatorId | ACTIVITY_FLAG_ISREGISTERED);
-    }
-
-    virtual void releaseRowset(unsigned count, byte * * rowset)
-    {
-        rtlReleaseRowset(count, rowset);
-    }
-
-    virtual byte * * linkRowset(byte * * rowset)
-    {
-        return rtlLinkRowset(rowset);
-    }
-
-    virtual byte * * appendRowOwn(byte * * rowset, unsigned newRowCount, void * row)
-    {
-        if (!rowset)
-            rowset = createRowset(newRowCount);
-        else
-        {
-            size32_t capacity;
-            rowset = (byte * *)rowManager.resizeRow(rowset, (newRowCount-1) * sizeof(void *), newRowCount * sizeof(void *), allocatorId | ACTIVITY_FLAG_ISREGISTERED, capacity);
-        }
-        rowset[newRowCount-1] = (byte *)row;
-        return rowset;
-    }
-
-    virtual byte * * reallocRows(byte * * rowset, unsigned oldRowCount, unsigned newRowCount)
-    {
-        if (!rowset)
-            rowset = createRowset(newRowCount);
-        else
-        {
-            size32_t capacity;
-            rowset = (byte * *)rowManager.resizeRow(rowset, oldRowCount * sizeof(void *), newRowCount * sizeof(void *), allocatorId | ACTIVITY_FLAG_ISREGISTERED, capacity);
-        }
-        //New rows (if any) aren't cleared....
-        return rowset;
-    }
-
-    virtual void * createRow()
-    {
-        const size32_t allocSize = meta.getInitialSize();
-        return rowManager.allocate(allocSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED);
-    }
-
-    virtual void * createRow(size32_t & allocatedSize)
-    {
-        const size32_t allocSize = meta.getInitialSize();
-        void *ret = rowManager.allocate(allocSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED);
-        allocatedSize = allocSize;
-        return ret;
-    }
-
-    virtual void releaseRow(const void * row)
-    {
-        ReleaseRoxieRow(row);
-    }
-
-    virtual void * linkRow(const void * row)
-    {
-        LinkRoxieRow(row);
-        return const_cast<void *>(row);
-    }
-
-    virtual void * resizeRow(size32_t newSize, void * row, size32_t & size)
-    {
-        size32_t capacity;
-        void * ret = rowManager.resizeRow(row, size, newSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED, capacity);
-        size = capacity;
-        return ret;
-    }
-
-    virtual void * finalizeRow(size32_t newSize, void * row, size32_t oldSize)
-    {
-        unsigned id = allocatorId | ACTIVITY_FLAG_ISREGISTERED;
-        if (meta.needsDestruct()) id |= ACTIVITY_FLAG_NEEDSDESTRUCTOR;
-        return rowManager.finalizeRow(row, oldSize, newSize, id);
-    }
-
-    virtual IOutputMetaData * queryOutputMeta()
-    {
-        return meta.queryOriginal();
-    }
-    virtual unsigned queryActivityId()
-    {
-        return activityId;
-    }
-    virtual StringBuffer &getId(StringBuffer &idStr)
-    {
-        return idStr.append(activityId); // MORE - may want more context info in here
-    }
-    virtual IOutputRowSerializer *createRowSerializer(ICodeContext *ctx)
-    {
-        return meta.createRowSerializer(ctx, activityId);
-    }
-    virtual IOutputRowDeserializer *createRowDeserializer(ICodeContext *ctx)
-    {
-        return meta.createRowDeserializer(ctx, activityId);
-    }
-protected:
-    roxiemem::IRowManager & rowManager;
-    CachedOutputMetaData meta;
-    unsigned activityId;
-    unsigned allocatorId;
-};
-
 //---------------------------------------------------------------------------
 
 class CHThorException : public CInterface, implements IHThorException

roxie/ccd/ccdquery.hpp

@@ -30,6 +30,7 @@
 #include "thorcommon.hpp"
 #include "ccddali.hpp"
 #include "thorcommon.ipp"
+#include "roxierow.hpp"
 
 class TranslatorArray : public CInterface, implements IInterface
 {
@@ -206,218 +207,6 @@ public:
 };
 
 
-class RoxieEngineRowAllocatorBase : public CInterface, implements IEngineRowAllocator
-{
-public:
-    RoxieEngineRowAllocatorBase(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId)
-        : rowManager(_rowManager), meta(_meta) 
-    {
-        activityId = _activityId;
-        allocatorId = _allocatorId;
-    }
-
-    IMPLEMENT_IINTERFACE
-
-//interface IEngineRowsetAllocator
-    virtual byte * * createRowset(unsigned count)
-    {
-        if (count == 0)
-            return NULL;
-        return (byte **) rowManager.allocate(count * sizeof(void *), allocatorId | ACTIVITY_FLAG_ISREGISTERED);
-    }
-
-    virtual void releaseRowset(unsigned count, byte * * rowset)
-    {
-        rtlReleaseRowset(count, rowset);
-    }
-
-    virtual byte * * linkRowset(byte * * rowset)
-    {
-        return rtlLinkRowset(rowset);
-    }
-
-    virtual byte * * appendRowOwn(byte * * rowset, unsigned newRowCount, void * row)
-    {
-        byte * * expanded = doReallocRows(rowset, newRowCount-1, newRowCount);
-        expanded[newRowCount-1] = (byte *)row;
-        return expanded;
-    }
-
-    virtual byte * * reallocRows(byte * * rowset, unsigned oldRowCount, unsigned newRowCount)
-    {
-        //New rows (if any) aren't cleared....
-        return doReallocRows(rowset, oldRowCount, newRowCount);
-    }
-
-    inline byte * * doReallocRows(byte * * rowset, unsigned oldRowCount, unsigned newRowCount)
-    {
-        if (!rowset)
-            return createRowset(newRowCount);
-
-        if (newRowCount * sizeof(void *) <= RoxieRowCapacity(rowset))
-            return rowset;
-
-        size32_t capacity;
-        return (byte * *)rowManager.resizeRow(rowset, oldRowCount * sizeof(void *), newRowCount * sizeof(void *), allocatorId | ACTIVITY_FLAG_ISREGISTERED, capacity);
-    }
-
-    virtual void releaseRow(const void * row)
-    {
-        ReleaseRoxieRow(row);
-    }
-
-    virtual void * linkRow(const void * row)
-    {
-        LinkRoxieRow(row);
-        return const_cast<void *>(row);
-    }
-
-    virtual IOutputMetaData * queryOutputMeta()
-    {
-        return meta.queryOriginal();
-    }
-    virtual unsigned queryActivityId()
-    {
-        return activityId;
-    }
-    virtual StringBuffer &getId(StringBuffer &idStr)
-    {
-        return idStr.append(activityId); // MORE - may want more context info in here
-    }
-    virtual IOutputRowSerializer *createRowSerializer(ICodeContext *ctx)
-    {
-        return meta.createRowSerializer(ctx, activityId);
-    }
-    virtual IOutputRowDeserializer *createRowDeserializer(ICodeContext *ctx)
-    {
-        return meta.createRowDeserializer(ctx, activityId);
-    }
-protected:
-    roxiemem::IRowManager & rowManager;
-    const CachedOutputMetaData meta;
-    unsigned activityId;
-    unsigned allocatorId;
-};
-
-//General purpose row allocator here for reference - should be removed once spcialised versions are created
-class RoxieEngineRowAllocator : public RoxieEngineRowAllocatorBase
-{
-public:
-    RoxieEngineRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId)
-        : RoxieEngineRowAllocatorBase(_rowManager, _meta, _activityId, _allocatorId)
-    {
-    }
-
-    virtual void * createRow()
-    {
-        size32_t allocSize = meta.getInitialSize();
-        return rowManager.allocate(allocSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED);
-    }
-
-    virtual void * createRow(size32_t & allocatedSize)
-    {
-        const size32_t allocSize = meta.getInitialSize();
-        void *ret = rowManager.allocate(allocSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED);
-        //more: allocate could return the allocated size, but that would penalise the fixed row case
-        allocatedSize = RoxieRowCapacity(ret);
-        return ret;
-    }
-
-    virtual void * resizeRow(size32_t newSize, void * row, size32_t & size)
-    {
-        return rowManager.resizeRow(row, size, newSize, allocatorId | ACTIVITY_FLAG_ISREGISTERED, size);
-    }
-
-    virtual void * finalizeRow(size32_t finalSize, void * row, size32_t oldSize)
-    {
-        unsigned id = allocatorId | ACTIVITY_FLAG_ISREGISTERED;
-        if (meta.needsDestruct()) id |= ACTIVITY_FLAG_NEEDSDESTRUCTOR;
-        return rowManager.finalizeRow(row, oldSize, finalSize, id);
-    }
-};
-
-class RoxieEngineFixedRowAllocator : public RoxieEngineRowAllocatorBase
-{
-public:
-    RoxieEngineFixedRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId, bool packed)
-        : RoxieEngineRowAllocatorBase(_rowManager, _meta, _activityId, _allocatorId)
-    {
-        unsigned flags = packed ? roxiemem::RHFpacked : roxiemem::RHFnone;
-        if (meta.needsDestruct())
-            flags |= roxiemem::RHFhasdestructor;
-        heap.setown(rowManager.createFixedRowHeap(meta.getFixedSize(), activityId | ACTIVITY_FLAG_ISREGISTERED, (roxiemem::RoxieHeapFlags)flags));
-    }
-
-    virtual void * createRow()
-    {
-        return heap->allocate();
-    }
-
-    virtual void * createRow(size32_t & allocatedSize)
-    {
-        allocatedSize = meta.getInitialSize();
-        return heap->allocate();
-    }
-
-    virtual void * resizeRow(size32_t newSize, void * row, size32_t & size)
-    {
-        throwUnexpected();
-        return NULL;
-    }
-
-    virtual void * finalizeRow(size32_t finalSize, void * row, size32_t oldSize)
-    {
-        if (!meta.needsDestruct())
-            return row;
-        return heap->finalizeRow(row);
-    }
-
-protected:
-    Owned<roxiemem::IFixedRowHeap> heap;
-};
-
-class RoxieEngineVariableRowAllocator : public RoxieEngineRowAllocatorBase
-{
-public:
-    RoxieEngineVariableRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId, bool _packed)
-        : RoxieEngineRowAllocatorBase(_rowManager, _meta, _activityId, _allocatorId), packed(_packed)
-    {
-        unsigned flags = packed ? roxiemem::RHFpacked : roxiemem::RHFnone;
-        if (meta.needsDestruct())
-            flags |= roxiemem::RHFhasdestructor;
-        heap.setown(rowManager.createVariableRowHeap(activityId | ACTIVITY_FLAG_ISREGISTERED, (roxiemem::RoxieHeapFlags)flags));
-    }
-
-    virtual void * createRow()
-    {
-        size32_t allocSize = meta.getInitialSize();
-        size32_t capacity;
-        return heap->allocate(allocSize, capacity);
-    }
-
-    virtual void * createRow(size32_t & allocatedSize)
-    {
-        const size32_t allocSize = meta.getInitialSize();
-        return heap->allocate(allocSize, allocatedSize);
-    }
-
-    virtual void * resizeRow(size32_t newSize, void * row, size32_t & size)
-    {
-        return heap->resizeRow(row, size, newSize, size);
-    }
-
-    virtual void * finalizeRow(size32_t finalSize, void * row, size32_t oldSize)
-    {
-        if (!meta.needsDestruct() && !packed)
-            return row;
-        return heap->finalizeRow(row, oldSize, finalSize);
-    }
-
-protected:
-    Owned<roxiemem::IVariableRowHeap> heap;
-    bool packed;    // may not be needed - depends on implementation
-};
-
 interface IQueryDll : public IInterface
 {
     virtual HelperFactory *getFactory(const char *name) const = 0;

roxie/ccd/ccdqueue.cpp

@@ -35,8 +35,7 @@
 #endif
 
 #ifdef _DEBUG
-#define allocate(a) allocate(a, __LINE__)
-#define clone(a,b) clone(a, b, __LINE__)
+#define allocate(a, x) allocate(a, __LINE__)
 #endif
 
 CriticalSection ibytiCrit; // CAUTION - not safe to use spinlocks as real-time thread accesses

roxie/ccd/ccdserver.cpp

@@ -66,6 +66,8 @@ namespace ccdserver_hqlhelper
 #include "nbcd.hpp"
 #include "roxiehelper.hpp"
 #include "roxielmj.hpp"
+#include "roxierow.hpp"
+
 #include "thorplugin.hpp"
 #include "keybuild.hpp"
 
@@ -28063,7 +28065,7 @@ public:
     {
         // MORE - may need to do some caching/commoning up here otherwise GRAPH in a child query may use too many
         SpinBlock b(allAllocatorsLock);
-        IEngineRowAllocator * ret = new RoxieEngineRowAllocator(*rowManager, meta, activityId, allAllocators.ordinality());
+        IEngineRowAllocator * ret = createRoxieRowAllocator(*rowManager, meta, activityId, allAllocators.ordinality(), false);
         LINK(ret);
         allAllocators.append(*ret);
         return ret;

roxie/roxiemem/roxiemem.cpp

@@ -1014,6 +1014,50 @@ public:
 
 //================================================================================
 //
+class CRoxieFixedRowHeap : implements IFixedRowHeap, public CInterface
+{
+public:
+    CRoxieFixedRowHeap(CChunkingRowManager * _rowManager, size32_t _fixedSize, unsigned _activityId, RoxieHeapFlags _flags)
+        : rowManager(_rowManager), fixedSize(_fixedSize), activityId(_activityId), flags(_flags)
+    {
+    }
+    IMPLEMENT_IINTERFACE
+
+    virtual void *allocate();
+    virtual void *finalizeRow(void *final);
+
+protected:
+    Linked<CChunkingRowManager> rowManager;
+    size32_t fixedSize;
+    unsigned activityId;
+    RoxieHeapFlags flags;
+};
+
+
+//================================================================================
+//
+class CRoxieVariableRowHeap : implements IVariableRowHeap, public CInterface
+{
+public:
+    CRoxieVariableRowHeap(CChunkingRowManager * _rowManager, unsigned _activityId, RoxieHeapFlags _flags)
+        : rowManager(_rowManager), activityId(_activityId), flags(_flags)
+    {
+    }
+    IMPLEMENT_IINTERFACE
+
+    virtual void *allocate(size32_t size, size32_t & capacity);
+    virtual void *resizeRow(void * original, size32_t oldsize, size32_t newsize, size32_t &capacity);
+    virtual void *finalizeRow(void *final, size32_t originalSize, size32_t finalSize);
+
+protected:
+    Linked<CChunkingRowManager> rowManager;
+    unsigned activityId;
+    RoxieHeapFlags flags;
+};
+
+
+//================================================================================
+//
 class CChunkingRowManager : public CInterface, implements IRowManager
 {
     friend class CRoxieFixedRowHeap;
@@ -1537,84 +1581,51 @@ public:
     virtual IFixedRowHeap * createFixedRowHeap(size32_t fixedSize, unsigned activityId, RoxieHeapFlags flags)
     {
         //Although the activityId is passed here, there is nothing to stop multiple RowHeaps sharing the same ChunkAllocator
-        return NULL;
+        return new CRoxieFixedRowHeap(this, fixedSize, activityId, flags);
     }
 
     virtual IVariableRowHeap * createVariableRowHeap(unsigned activityId, RoxieHeapFlags flags)
     {
         //Although the activityId is passed here, there is nothing to stop multiple RowHeaps sharing the same ChunkAllocator
-        return NULL;
+        return new CRoxieVariableRowHeap(this, activityId, flags);
     }
 };
 
-class CRoxieFixedRowHeap : implements IFixedRowHeap, public CInterface
-{
-public:
-    CRoxieFixedRowHeap(CChunkingRowManager * _rowManager, size32_t _fixedSize, unsigned _activityId, RoxieHeapFlags _flags)
-        : rowManager(_rowManager), fixedSize(_fixedSize), activityId(_activityId), flags(_flags)
-    {
-    }
-    IMPLEMENT_IINTERFACE
-
-    virtual void *allocate()
-    {
-        return rowManager->allocate(fixedSize, activityId);
-    }
-
-    virtual void *finalizeRow(void *final)
-    {
-        //MORE: Checking heap checks for not shared.
-        if (flags & RHFhasdestructor)
-            HeapletBase::setDestructorFlag(final);
-        return final;
-    }
 
-protected:
-    Linked<CChunkingRowManager> rowManager;
-    size32_t fixedSize;
-    unsigned activityId;
-    RoxieHeapFlags flags;
-};
-
-
-class CRoxieVariableRowHeap : implements IFixedRowHeap, public CInterface
+void * CRoxieFixedRowHeap::allocate()
 {
-public:
-    CRoxieVariableRowHeap(CChunkingRowManager * _rowManager, unsigned _activityId, RoxieHeapFlags _flags)
-        : rowManager(_rowManager), activityId(_activityId), flags(_flags)
-    {
-    }
-    IMPLEMENT_IINTERFACE
-
-    virtual void *allocate(size32_t size, size32_t & capacity)
-    {
-        void * ret = rowManager->allocate(fixedSize, activityId);
-        capacity = RoxieRowCapacity(ret);
-        return ret;
-    }
+    return rowManager->allocate(fixedSize, activityId);
+}
 
-    virtual void *resizeRow(void * original, size32_t oldsize, size32_t newsize, size32_t &capacity)
-    {
-        return rowManager->resizeRow(original, oldsize, newsize, activityId, capacity);
-    }
+void * CRoxieFixedRowHeap::finalizeRow(void *final)
+{
+    //MORE: Checking heap checks for not shared.
+    if (flags & RHFhasdestructor)
+        HeapletBase::setDestructorFlag(final);
+    return final;
+}
 
-    virtual void *finalizeRow(void *final, size32_t originalSize, size32_t finalSize)
-    {
-        return rowManager->finalizeRow(final, originalSize, finalSize, activityId);
-        //If never shrink the following should be sufficient.
-        //MORE: Checking heap checks for not shared.
-        if (flags & RHFhasdestructor)
-            HeapletBase::setDestructorFlag(final);
-        return final;
-    }
+void * CRoxieVariableRowHeap::allocate(size32_t size, size32_t & capacity)
+{
+    void * ret = rowManager->allocate(size, activityId);
+    capacity = RoxieRowCapacity(ret);
+    return ret;
+}
 
-protected:
-    Linked<CChunkingRowManager> rowManager;
-    size32_t fixedSize;
-    unsigned activityId;
-    RoxieHeapFlags flags;
-};
+void * CRoxieVariableRowHeap::resizeRow(void * original, size32_t oldsize, size32_t newsize, size32_t &capacity)
+{
+    return rowManager->resizeRow(original, oldsize, newsize, activityId, capacity);
+}
 
+void * CRoxieVariableRowHeap::finalizeRow(void *final, size32_t originalSize, size32_t finalSize)
+{
+    return rowManager->finalizeRow(final, originalSize, finalSize, activityId);
+    //If never shrink the following should be sufficient.
+    //MORE: Checking heap checks for not shared.
+    if (flags & RHFhasdestructor)
+        HeapletBase::setDestructorFlag(final);
+    return final;
+}
 
 //================================================================================
 // Buffer manager - blocked 

rtl/eclrtl/rtlds.cpp

@@ -272,13 +272,17 @@ byte * RtlDynamicRowBuilder::ensureCapacity(size32_t required, const char * fiel
     {
         if (!self)
             create();
-        void * next = rowAllocator->resizeRow(required, self, maxLength);
-        if (!next)
+
+        if (required > maxLength)
         {
-            rtlReportFieldOverflow(required, maxLength, fieldName);
-            return NULL;
-        }   
-        self = static_cast<byte *>(next);
+            void * next = rowAllocator->resizeRow(required, self, maxLength);
+            if (!next)
+            {
+                rtlReportFieldOverflow(required, maxLength, fieldName);
+                return NULL;
+            }
+            self = static_cast<byte *>(next);
+        }
     }
     return self;
 }