HPCC-16854 Start exploring classes for dynamically accessing rows

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

rtl/eclrtl/CMakeLists.txt

@@ -38,6 +38,7 @@ set (    SRCS
          rtlrank.cpp 
          rtlfield.cpp 
          rtlread.cpp 
+         rtlrecord.cpp
          rtltype.cpp 
          rtlxml.cpp
          
@@ -52,6 +53,7 @@ set (    SRCS
          rtlkey2.hpp
          rtlkey.hpp
          rtlread_imp.hpp
+         rtlrecord.hpp
          rtlsize.hpp
          rtltype.hpp
          rtlbcdtest.cpp 

rtl/eclrtl/rtldynfield.hpp

@@ -26,16 +26,23 @@ struct ECLRTL_API RtlDynFieldInfo : public RtlFieldInfo
 {
 public:
     RtlDynFieldInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type)
-    : RtlFieldInfo(nullptr, nullptr, _type, nullptr), nameAttr(_name), xpathAttr(_xpath)
+    : RtlFieldInfo(_name, _xpath, _type, nullptr)
     {
-        name = nameAttr.get();
-        xpath = xpathAttr.get();
     }
-
-protected:
-    StringAttr nameAttr;
-    StringAttr xpathAttr;
+    ~RtlDynFieldInfo()
+    {
+        free(const_cast<char *>(name));
+        free(const_cast<char *>(xpath));
+    }
 };
 
 
+//-------------------------------------------------------------------------------------------------------------------
+
+struct ECLRTL_API RtlDynRecordTypeInfo : public RtlRecordTypeInfo
+{
+    inline RtlDynRecordTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlRecordTypeInfo(_fieldType, _length, _fields) {}
+    ~RtlDynRecordTypeInfo() { delete[] fields; }
+};
+
 #endif

rtl/eclrtl/rtlfield.cpp

@@ -89,6 +89,26 @@ public:
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t ECLRTL_API getMinSize(const RtlFieldInfo * const * fields)
+{
+    size32_t minSize = 0;
+    loop
+    {
+        const RtlFieldInfo * cur = *fields;
+        if (!cur)
+            return minSize;
+        minSize += cur->type->getMinSize();
+        fields++;
+    }
+}
+
+//-------------------------------------------------------------------------------------------------------------------
+
+size32_t RtlTypeInfoBase::getMinSize() const
+{
+    return length;
+}
+
 size32_t RtlTypeInfoBase::size(const byte * self, const byte * selfrow) const 
 {
     return length; 
@@ -301,6 +321,11 @@ __int64 RtlSwapIntTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlPackedIntTypeInfo::getMinSize() const
+{
+    return 1;
+}
+
 size32_t RtlPackedIntTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     return rtlGetPackedSize(self); 
@@ -353,6 +378,13 @@ __int64 RtlPackedIntTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlStringTypeInfo::getMinSize() const
+{
+    if (isFixedSize())
+        return length;
+    return sizeof(size32_t);
+}
+
 size32_t RtlStringTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isFixedSize())
@@ -479,6 +511,13 @@ __int64 RtlStringTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlDataTypeInfo::getMinSize() const
+{
+    if (isFixedSize())
+        return length;
+    return sizeof(size32_t);
+}
+
 size32_t RtlDataTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isFixedSize())
@@ -576,6 +615,13 @@ __int64 RtlDataTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlVarStringTypeInfo::getMinSize() const
+{
+    if (isFixedSize())
+        return length;
+    return 1;
+}
+
 size32_t RtlVarStringTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isFixedSize())
@@ -669,6 +715,13 @@ __int64 RtlVarStringTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlQStringTypeInfo::getMinSize() const
+{
+    if (isFixedSize())
+        return rtlQStrSize(length);
+    return sizeof(size32_t);
+}
+
 size32_t RtlQStringTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isFixedSize())
@@ -775,6 +828,11 @@ size32_t RtlDecimalTypeInfo::calcSize() const
     return (getDecimalDigits()+2)/2;
 }
 
+size32_t RtlDecimalTypeInfo::getMinSize() const
+{
+    return calcSize();
+}
+
 size32_t RtlDecimalTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     return calcSize();
@@ -879,6 +937,13 @@ __int64 RtlCharTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlUnicodeTypeInfo::getMinSize() const
+{
+    if (isFixedSize())
+        return length * sizeof(UChar);
+    return sizeof(size32_t);
+}
+
 size32_t RtlUnicodeTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isFixedSize())
@@ -980,6 +1045,13 @@ __int64 RtlUnicodeTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlVarUnicodeTypeInfo::getMinSize() const
+{
+    if (isFixedSize())
+        return length * sizeof(UChar);
+    return sizeof(UChar);
+}
+
 size32_t RtlVarUnicodeTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isFixedSize())
@@ -1056,6 +1128,11 @@ __int64 RtlVarUnicodeTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlUtf8TypeInfo::getMinSize() const
+{
+    return sizeof(size32_t);
+}
+
 size32_t RtlUtf8TypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     assertex(!isFixedSize());
@@ -1185,6 +1262,11 @@ inline size32_t toXMLFields(const RtlFieldInfo * const * cur, const byte * self,
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlRecordTypeInfo::getMinSize() const
+{
+    return ::getMinSize(fields);
+}
+
 size32_t RtlRecordTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     return sizeFields(fields, self, self);
@@ -1330,6 +1412,11 @@ __int64 RtlEndRowTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlSetTypeInfo::getMinSize() const
+{
+    return sizeof(bool) + sizeof(size32_t);
+}
+
 size32_t RtlSetTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     return sizeof(bool) + sizeof(size32_t) + rtlReadUInt4(self + sizeof(bool));
@@ -1433,6 +1520,13 @@ size32_t RtlSetTypeInfo::toXML(const byte * self, const byte * selfrow, const Rt
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlRowTypeInfo::getMinSize() const
+{
+    if (isLinkCounted())
+        return sizeof(void *);
+    return child->getMinSize();
+}
+
 size32_t RtlRowTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isLinkCounted())
@@ -1466,6 +1560,13 @@ size32_t RtlRowTypeInfo::toXML(const byte * self, const byte * selfrow, const Rt
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlDatasetTypeInfo::getMinSize() const
+{
+    if (isLinkCounted())
+        return sizeof(size32_t) + sizeof(void * *);
+    return sizeof(size32_t);
+}
+
 size32_t RtlDatasetTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (isLinkCounted())
@@ -1601,6 +1702,13 @@ size32_t RtlDatasetTypeInfo::toXML(const byte * self, const byte * selfrow, cons
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlDictionaryTypeInfo::getMinSize() const
+{
+    if (isLinkCounted())
+        return sizeof(size32_t) + sizeof(void * *);
+    return sizeof(size32_t);
+}
+
 size32_t RtlDictionaryTypeInfo::size(const byte * self, const byte * selfrow) const
 {
     if (isLinkCounted())
@@ -1701,6 +1809,11 @@ size32_t RtlDictionaryTypeInfo::toXML(const byte * self, const byte * selfrow, c
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlIfBlockTypeInfo::getMinSize() const
+{
+    return 0;
+}
+
 size32_t RtlIfBlockTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (getCondition(selfrow))
@@ -1760,6 +1873,13 @@ unsigned __int64 RtlBitfieldTypeInfo::unsignedValue(const void * self) const
 }
 
 
+size32_t RtlBitfieldTypeInfo::getMinSize() const
+{
+    if (fieldType & RFTMislastbitfield)
+        return getBitfieldIntSize();
+    return 0;
+}
+
 size32_t RtlBitfieldTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     if (fieldType & RFTMislastbitfield)
@@ -1807,6 +1927,12 @@ __int64 RtlBitfieldTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlUnimplementedTypeInfo::getMinSize() const
+{
+    rtlFailUnexpected();
+    return 0;
+}
+
 size32_t RtlUnimplementedTypeInfo::size(const byte * self, const byte * selfrow) const 
 {
     rtlFailUnexpected();

rtl/eclrtl/rtlfield.hpp

@@ -29,11 +29,14 @@ in that context.  For that reason the classes have no destructors.
 The file rtldynfield contains classes which manage instances of these classes which are dynamically created.
 */
 
+size32_t ECLRTL_API getMinSize(const RtlFieldInfo * const * fields);
+
 // A base implementation of RtlTypeInfo
 struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
 {
     inline RtlTypeInfoBase(unsigned _fieldType, unsigned _length) : RtlTypeInfo(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
@@ -98,6 +101,7 @@ struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
 {
     inline RtlPackedIntTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -110,6 +114,7 @@ struct ECLRTL_API RtlStringTypeInfo : public RtlTypeInfoBase
 {
     inline RtlStringTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -122,6 +127,7 @@ struct ECLRTL_API RtlDataTypeInfo : public RtlTypeInfoBase
 {
     inline RtlDataTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -134,6 +140,7 @@ struct ECLRTL_API RtlVarStringTypeInfo : public RtlTypeInfoBase
 {
     inline RtlVarStringTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -146,6 +153,7 @@ struct ECLRTL_API RtlQStringTypeInfo : public RtlTypeInfoBase
 {
     inline RtlQStringTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -158,6 +166,7 @@ struct ECLRTL_API RtlDecimalTypeInfo : public RtlTypeInfoBase
 {
     inline RtlDecimalTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -184,6 +193,7 @@ struct ECLRTL_API RtlUnicodeTypeInfo : public RtlTypeInfoBase
 public:
     inline RtlUnicodeTypeInfo(unsigned _fieldType, unsigned _length, const char * _locale) : RtlTypeInfoBase(_fieldType, _length), locale(_locale) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -202,6 +212,7 @@ struct ECLRTL_API RtlVarUnicodeTypeInfo : public RtlTypeInfoBase
 public:
     inline RtlVarUnicodeTypeInfo(unsigned _fieldType, unsigned _length, const char * _locale) : RtlTypeInfoBase(_fieldType, _length), locale(_locale) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -220,6 +231,7 @@ struct ECLRTL_API RtlUtf8TypeInfo : public RtlTypeInfoBase
 public:
     inline RtlUtf8TypeInfo(unsigned _fieldType, unsigned _length, const char * _locale) : RtlTypeInfoBase(_fieldType, _length), locale(_locale) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -238,6 +250,7 @@ struct ECLRTL_API RtlRecordTypeInfo : public RtlTypeInfoBase
     inline RtlRecordTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlTypeInfoBase(_fieldType, _length), fields(_fields) {}
     const RtlFieldInfo * const * fields;                // null terminated
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -289,6 +302,7 @@ struct ECLRTL_API RtlSetTypeInfo : public RtlCompoundTypeInfo
 {
     inline RtlSetTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo * _child) : RtlCompoundTypeInfo(_fieldType, _length, _child) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -299,6 +313,7 @@ struct ECLRTL_API RtlRowTypeInfo : public RtlCompoundTypeInfo
 {
     inline RtlRowTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo * _child) : RtlCompoundTypeInfo(_fieldType, _length, _child) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
@@ -309,6 +324,7 @@ struct ECLRTL_API RtlDatasetTypeInfo : public RtlCompoundTypeInfo
 {
     inline RtlDatasetTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo * _child) : RtlCompoundTypeInfo(_fieldType, _length, _child) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -322,6 +338,7 @@ struct ECLRTL_API RtlDictionaryTypeInfo : public RtlCompoundTypeInfo
     : RtlCompoundTypeInfo(_fieldType, _length, _child), hashInfo(_hashInfo) {}
     IHThorHashLookupInfo * hashInfo;
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
@@ -335,6 +352,7 @@ struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
     const RtlFieldInfo * const * fields;                // null terminated
 
     virtual bool getCondition(const byte * selfrow) const = 0;
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
@@ -349,6 +367,7 @@ struct ECLRTL_API RtlBitfieldTypeInfo : public RtlTypeInfoBase
 {
     inline RtlBitfieldTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
@@ -364,6 +383,7 @@ struct ECLRTL_API RtlUnimplementedTypeInfo : public RtlTypeInfoBase
 {
     inline RtlUnimplementedTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
+    virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;

rtl/eclrtl/rtlrecord.cpp

@@ -0,0 +1,207 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include <math.h>
+#include <stdio.h>
+#include "jmisc.hpp"
+#include "jlib.hpp"
+#include "eclhelper.hpp"
+#include "eclrtl_imp.hpp"
+#include "rtlrecord.hpp"
+
+/*
+ * Potential different implementations (for all fixed size has no penalty):
+ *
+ * a. when row changes, update offsets of all offset fields
+ *    + access is simple, single array lookup
+ *    - lots more fields are updated even if the fields aren't used.
+ *
+ * b. when a row changes update the next offset for all variable size fields.
+ *    + minimal updates
+ *    - offset access is more complex
+ *
+ * c. when row changes clear cache, and calculate on demand.
+ *    + trivial update on row change
+ *    + no cost of fields not accessed
+ *    + may be required to implement ifblocks - since fields in the test expression must be evaluated before all the
+ *      offsets are known.  Depends on the implementation of ifblocks!
+ *    - accessing an offset will involve a loop and be more expensive
+ *
+ * Handling complications:
+ * a. Nested rows
+ *    A dilemma.  Should they be expanded?
+ *    + If they are expanded then it makes it much simpler to use externally.
+ *    - The nested fields have compound names storing and matching them is a complication.
+ *    - The code generator doesn't expand.
+ *    + It is easier to calculate offsets since all fields are supported by the same class.
+ *    + Sizes for unexpanded rows would need their own size caching classes.  That is more complex!
+ *    + The meta information currently processes child information.
+ *
+ * Not expanding is complicated if you also try and only calculate the offsets of fields in nested records once - since you end
+ * up needing to call back and forth between instances and the static information.
+ * However, if nested records are processed by using size(), and selections from them are processed by instantiating
+ * an instance of the nested row it is all much simpler.  The cost is potentially re-evaluating sizes of nested fields.  Potentially
+ * inefficient if most are fixed size, but some are variable.
+ *
+ * b. bitfields
+ *    the last bitfield in a bitfield container has the size of the container, the others have 0 size.
+ *
+ * c. ifblocks
+ *    Nasty.  Allowing direct access means the flag would need checking, and a field would need a pointer
+ *    to its containing ifblock.  Cleanest to have a different derived implementation which was used if the record
+ *    contained ifblocks which added calls to check ifblocks before size()/getValue() etc.
+ *    Will require an extra row parameter to every RtlTypeInfo::getX() function.
+ *    Evaluating the test expression without compiling will require expression interpreting.
+ *
+ * d. alien datatypes
+ *    As long as the rtlField class implements the functions then it shouldn't cause any problems.  Evaluating at
+ *    from a record at runtime without compiling will be tricky - requires an interpreter.
+ *
+ * Other
+ *  Add a minSize to each field (unless already stored in the record information)
+ *
+ * Expression interpreting:
+ *   Replace the no_select with a CHqlBoundExpr(no_select, fieldid).
+ *   Evaluate (ctx [ logical->RtlFieldOffsetCalculator mapping ]).
+ *   Even better if mapped direct to something that represents the base cursor so no need to search ctx.
+ *   For nested selects the code would need to be consistent.
+ */
+
+static unsigned countFields(const RtlFieldInfo * const * fields)
+{
+    unsigned cnt = 0;
+    for (;*fields;fields++)
+        cnt++;
+    return cnt;
+}
+
+
+RtlRecord::RtlRecord(const RtlRecordTypeInfo & record) : fields(record.fields)
+{
+    //MORE: Does not cope with ifblocks.
+    numVarFields = 0;
+    numFields = countFields(fields);
+    for (unsigned i=0; i < numFields; i++)
+    {
+        if (!queryType(i)->isFixedSize())
+            numVarFields++;
+    }
+
+    fixedOffsets = new size_t[numFields + 1];
+    whichVariableOffset = new unsigned[numFields + 1];
+    variableFieldIds = new unsigned[numVarFields];
+
+    unsigned curVariable = 0;
+    size_t fixedOffset = 0;
+    for (unsigned i=0;; i++)
+    {
+        whichVariableOffset[i] = curVariable;
+        fixedOffsets[i] = fixedOffset;
+        if (i == numFields)
+            break;
+
+        const RtlTypeInfo * curType = queryType(i);
+        if (curType->isFixedSize())
+        {
+            size_t thisSize = curType->size(nullptr, nullptr);
+            fixedOffset += thisSize;
+        }
+        else
+        {
+            variableFieldIds[curVariable] = i;
+            curVariable++;
+            fixedOffset = 0;
+        }
+    }
+}
+
+
+RtlRecord::~RtlRecord()
+{
+    delete [] fixedOffsets;
+    delete [] whichVariableOffset;
+    delete [] variableFieldIds;
+}
+
+
+void RtlRecord::calcRowOffsets(size_t * variableOffsets, const void * _row) const
+{
+    const byte * row = static_cast<const byte *>(_row);
+    for (unsigned i = 0; i < numVarFields; i++)
+    {
+        unsigned fieldIndex = variableFieldIds[i];
+        size_t offset = getOffset(variableOffsets, fieldIndex);
+        size_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
+        variableOffsets[i+1] = offset+fieldSize;
+    }
+}
+
+size32_t RtlRecord::getMinRecordSize() const
+{
+    if (numVarFields == 0)
+        return fixedOffsets[numFields];
+
+    size32_t minSize = 0;
+    for (unsigned i=0; i < numFields; i++)
+        minSize += queryType(i)->getMinSize();
+
+    return minSize;
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+RtlRow::RtlRow(const RtlRecord & _info, const void * optRow, size_t * _variableOffsets) : info(_info), variableOffsets(_variableOffsets)
+{
+    //variableOffset[0] is used for all fixed offset fields to avoid any special casing.
+    variableOffsets[0] = 0;
+    if (optRow)
+        setRow(optRow);
+}
+
+__int64 RtlRow::getInt(unsigned field) const
+{
+    const byte * self = reinterpret_cast<const byte *>(row);
+    const RtlTypeInfo * type = info.queryType(field);
+    return type->getInt(self + getOffset(field));
+}
+
+void RtlRow::getUtf8(size32_t & resultLen, char * & result, unsigned field) const
+{
+    const byte * self = reinterpret_cast<const byte *>(row);
+    const RtlTypeInfo * type = info.queryType(field);
+    return type->getUtf8(resultLen, result, self + getOffset(field));
+}
+
+void RtlRow::setRow(const void * _row)
+{
+    row = _row;
+    if (_row)
+        info.calcRowOffsets(variableOffsets, _row);
+}
+
+
+RtlDynRow::RtlDynRow(const RtlRecord & _info, const void * optRow) : RtlRow(_info, optRow, new size_t[_info.getNumVarFields()+1])
+{
+}
+
+RtlDynRow::~RtlDynRow()
+{
+    delete [] variableOffsets;
+}
+
+//---------------------------------------------------------------------------------------------------------------------

rtl/eclrtl/rtlrecord.hpp

@@ -0,0 +1,141 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef rtlrecord_hpp
+#define rtlrecord_hpp
+
+#include "rtlfield.hpp"
+
+//These classe provides a relatively efficient way to access fields within a variable length record structure.
+// Probably convert to an interface with various concrete implementations for varing degrees of complexity
+//
+// Complications:
+// * ifblocks
+// * nested records.
+// * alien data types
+//
+
+struct ECLRTL_API RtlRecord
+{
+public:
+    friend class RtlRow;
+    RtlRecord(const RtlRecordTypeInfo & fields);
+    ~RtlRecord();
+
+    void calcRowOffsets(size_t * variableOffsets, const void * _row) const;
+
+    virtual size32_t getFixedSize() const
+    {
+        return numVarFields ? 0 : fixedOffsets[numFields];
+    }
+
+    size_t getOffset(size_t * variableOffsets, unsigned field) const
+    {
+        return fixedOffsets[field] + variableOffsets[whichVariableOffset[field]];
+    }
+
+
+    size_t getRecordSize(size_t * variableOffsets) const
+    {
+        return getOffset(variableOffsets, numFields);
+    }
+
+    virtual size32_t getMinRecordSize() const;
+
+    inline unsigned getNumVarFields() const { return numVarFields; }
+    inline const RtlTypeInfo * queryType(unsigned field) const { return fields[field]->type; }
+
+protected:
+    size_t * fixedOffsets;        // fixed portion of the field offsets + 1 extra
+    unsigned * whichVariableOffset;// which variable offset should be added to the fixed
+    unsigned * variableFieldIds;  // map variable field to real field id.
+    unsigned numFields;
+    unsigned numVarFields;
+    const RtlFieldInfo * const * fields;
+};
+
+struct ECLRTL_API RtlRow
+{
+public:
+    RtlRow(const RtlRecord & _info, const void * optRow, size_t * _variableOffsets);
+
+    __int64 getInt(unsigned field) const;
+    void getUtf8(size32_t & resultLen, char * & result, unsigned field) const;
+
+    size_t getOffset(unsigned field) const
+    {
+        return info.getOffset(variableOffsets, field);
+    }
+
+    size_t getRecordSize() const
+    {
+        return info.getRecordSize(variableOffsets);
+    }
+
+    void setRow(const void * _row);
+
+protected:
+    const RtlRecord & info;
+    const void * row = nullptr;
+    size_t * variableOffsets;       // [0 + 1 entry for each variable size field ]
+};
+
+struct ECLRTL_API RtlDynRow : public RtlRow
+{
+public:
+    RtlDynRow(const RtlRecord & _info, const void * optRow = nullptr);
+    ~RtlDynRow();
+};
+
+//The following template class is used from the generated code to avoid allocating the offset array
+template <unsigned NUM_VARIABLE_FIELDS>
+struct ECLRTL_API RtlStaticRow : RtlRow
+{
+public:
+    RtlStaticRow(const RtlRecord & _info, const void * optRow = nullptr) : RtlRow(_info, optRow, &offsets) {}
+public:
+    size_t offsets[NUM_VARIABLE_FIELDS+1];
+};
+
+class ECLRTL_API RtlRecordSize : CInterfaceOf<IRecordSize>
+{
+    RtlRecordSize(const RtlRecordTypeInfo & fields) : offsetInformation(fields) {}
+
+    virtual size32_t getRecordSize(const void * row)
+    {
+        assertex(row);
+        //Allocate a temporary offset array on the stack to avoid runtime overhead.
+        size_t * variableOffsets = (size_t *)alloca((offsetInformation.getNumVarFields() + 1) * sizeof(size_t));
+        RtlRow offsetCalculator(offsetInformation, row, variableOffsets);
+        return offsetCalculator.getRecordSize();
+    }
+
+    virtual size32_t getFixedSize()
+    {
+        return offsetInformation.getFixedSize();
+    }
+    // returns 0 for variable row size
+    virtual size32_t getMinRecordSize() const
+    {
+        return offsetInformation.getMinRecordSize();
+    }
+
+protected:
+    RtlRecord offsetInformation;
+};
+
+#endif

rtl/include/eclhelper.hpp

@@ -357,6 +357,7 @@ interface RtlITypeInfo
 
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const = 0;
     virtual __int64 getInt(const void * ptr) const = 0;
+    virtual size32_t getMinSize() const = 0;
 };