HPCC-14700 Use a class to calculate offsets in rows with variable length fields

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/deftype/deftype.hpp

@@ -19,6 +19,7 @@
 #define _DEFTYPE_INCL
 
 #include "jcomp.hpp"
+#include "rtlconst.hpp"
 
 #ifdef DEFTYPE_EXPORTS
 #define DEFTYPE_API DECL_EXPORT
@@ -46,71 +47,6 @@ interface IHqlScope;
 #define type_littleendianint    type_swapint
 #endif
 
-// NOTE - do not change the values here - they are also used in Clarion and stored in a database!!
-// Add new types to the end
-
-enum type_vals 
-{
-    type_boolean        = 0, 
-    type_int            = 1,    
-    type_real           = 2, 
-    type_decimal        = 3, 
-    type_string         = 4, 
-    type_alias          = 5, // This is only used when serializing expression graphs
-    type_date           = 6, 
-type_unused2            = 7, 
-type_unused3            = 8, 
-    type_bitfield       = 9, 
-type_unused4            = 10, 
-    type_char           = 11,
-    type_enumerated     = 12, 
-    type_record         = 13, 
-    type_varstring      = 14,
-    type_blob           = 15, 
-    type_data           = 16, 
-    type_pointer        = 17, 
-    type_class          = 18, 
-    type_array          = 19,
-    type_table          = 20, 
-    type_set            = 21, 
-    type_row            = 22, 
-    type_groupedtable   = 23,
-    type_void           = 24,
-    type_alien          = 25,
-    type_swapint        = 26,
-    type_none           = 27,
-    type_packedint      = 28,
-type_unused5            = 29,
-    type_qstring        = 30,
-    type_unicode        = 31,
-    type_any            = 32,
-    type_varunicode     = 33,
-    type_pattern        = 34,
-    type_rule           = 35,
-    type_token          = 36,
-    type_feature        = 37,
-    type_event          = 38,
-    type_null           = 39,       // not the same as type_void, which should be reserved for actions.
-    type_scope          = 40,
-    type_utf8           = 41,
-    type_transform      = 42,
-    type_ifblock        = 43,       // not a real type -but used for the rtlfield serialization
-    type_function       = 44,
-    type_sortlist       = 45,
-    type_dictionary     = 46,
-
-    type_max,
-
-    type_modifier       = 0xff,     // used by getKind()
-    type_unsigned       = 0x100,  // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
-    type_ebcdic         = 0x200,   // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
-
-//Some pseudo types - never actually created
-    type_stringorunicode= 0xfc, // any string/unicode variant
-    type_numeric        = 0xfd,
-    type_scalar         = 0xfe,
-};
-
 enum typemod_t
 {
     typemod_none        = 0,

ecl/hql/hqlutil.cpp

@@ -2235,6 +2235,43 @@ unsigned getFlatFieldCount(IHqlExpression * expr)
 }
 
 
+unsigned getVarSizeFieldCount(IHqlExpression * expr, bool expandRows)
+{
+    //MORE: Is it worth caching the results of these functions as attributes?
+    switch (expr->getOperator())
+    {
+    case no_record:
+        {
+            unsigned count = 0;
+            ForEachChild(i, expr)
+                count += getVarSizeFieldCount(expr->queryChild(i), expandRows);
+            return count;
+        }
+    case no_ifblock:
+        return getVarSizeFieldCount(expr->queryChild(1), expandRows);
+    case no_field:
+        {
+            ITypeInfo * type = expr->queryType();
+            if (expandRows)
+            {
+                if (type->getTypeCode() == type_row)
+                    return getVarSizeFieldCount(expr->queryRecord(), expandRows);
+            }
+            if (isArrayRowset(type))
+                return 0;
+            return isUnknownSize(type) ? 1 : 0;
+        }
+    case no_attr:
+    case no_attr_link:
+    case no_attr_expr:
+        return 0;
+    default:
+        //UNIMPLEMENTED;
+        return 0;
+    }
+}
+
+
 unsigned isEmptyRecord(IHqlExpression * record)
 {
     ForEachChild(i, record)
@@ -4459,6 +4496,41 @@ bool containsIfBlock(IHqlExpression * record)
 }
 
 
+bool canCreateRtlTypeInfo(IHqlExpression * record)
+{
+    ForEachChild(i, record)
+    {
+        IHqlExpression * cur = record->queryChild(i);
+        switch (cur->getOperator())
+        {
+        case no_record:
+            if (!canCreateRtlTypeInfo(cur))
+                return false;
+            break;
+        case no_field:
+            switch (cur->queryType()->getTypeCode())
+            {
+            case type_row:
+                if (!canCreateRtlTypeInfo(cur->queryRecord()))
+                    return false;
+                break;
+            case type_table:
+            case type_groupedtable:
+                if (cur->hasAttribute(countAtom) || cur->hasAttribute(sizeofAtom))
+                    return false;
+                break;
+            case type_alien:
+                return false;
+            }
+            break;
+        case no_ifblock:
+            return false;
+        }
+    }
+    return true;
+}
+
+
 IHqlExpression * getFailCode(IHqlExpression * failExpr)
 {
     IHqlExpression * arg0 = failExpr->queryChild(0);

ecl/hql/hqlutil.hpp

@@ -90,6 +90,7 @@ extern HQL_API bool hasActiveTopDataset(IHqlExpression * expr);
 
 extern HQL_API unsigned getFieldCount(IHqlExpression * expr);
 extern HQL_API unsigned getFlatFieldCount(IHqlExpression * expr);
+extern HQL_API unsigned getVarSizeFieldCount(IHqlExpression * expr, bool expandRows);
 extern HQL_API unsigned isEmptyRecord(IHqlExpression * record);
 extern HQL_API unsigned isSimpleRecord(IHqlExpression * record);
 extern HQL_API void getSimpleFields(HqlExprArray &out, IHqlExpression *record);
@@ -341,6 +342,7 @@ extern HQL_API bool isUngroup(IHqlExpression * expr);
 extern HQL_API bool containsExpression(IHqlExpression * expr, IHqlExpression * search);
 extern HQL_API bool containsOperator(IHqlExpression * expr, node_operator search);
 extern HQL_API bool containsIfBlock(IHqlExpression * record);
+extern HQL_API bool canCreateRtlTypeInfo(IHqlExpression * record); // Can we generate completely valid rtltypeinfo?
 extern HQL_API IHqlExpression * removeAnnotations(IHqlExpression * expr, IHqlExpression * search);
 
 class HQL_API DependencyGatherer

ecl/hqlcpp/hqlcatom.cpp

@@ -26,6 +26,7 @@
 #include "hqlcatom.hpp"
 
 IAtom * _accessedFromChild_Atom;
+IAtom * accessorAtom;
 IAtom * activeActivityMarkerAtom;
 IAtom * activeMatchTextAtom;
 IAtom * activeMatchUnicodeAtom;
@@ -1429,6 +1430,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEID(xmlEncodeUStr);
 
 //Atoms
+    MAKEATOM(accessor);
     MAKEATOM(activeActivityMarker);
     MAKEATOM(activeMatchText);
     MAKEATOM(activeMatchUnicode);

ecl/hqlcpp/hqlcatom.hpp

@@ -24,6 +24,7 @@
 #endif
 
 extern IAtom * _accessedFromChild_Atom;
+extern IAtom * accessorAtom;
 extern IAtom * activeActivityMarkerAtom;
 extern IAtom * activeMatchTextAtom;
 extern IAtom * activeMatchUnicodeAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -254,6 +254,7 @@ bool isSimpleTranslatedStringExpr(IHqlExpression * expr)
         case no_constant:
         case no_variable:
         case no_callback:
+        case no_select:
             return true;
         case no_cast:
         case no_implicitcast:
@@ -1793,6 +1794,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.embeddedWarningsAsErrors,"embeddedWarningsFatal",true),
         DebugOption(options.optimizeCriticalFunctions,"optimizeCriticalFunctions",true),
         DebugOption(options.addLikelihoodToGraph,"addLikelihoodToGraph", true),
+        DebugOption(options.varFieldAccessorThreshold,"varFieldAccessorThreshold",3),   // Generate accessor classes for rows with #variable width fields >= threshold
     };
 
     //get options values from workunit
@@ -12261,9 +12263,8 @@ IHqlExpression * HqlCppTranslator::getBoundSize(const CHqlBoundExpr & bound)
             return getSizetConstant(sizeof(void*));
 
         IHqlExpression * record = ::queryRecord(type);
-        ColumnToOffsetMap * map = queryRecordOffsetMap(record);
-        if (map->isFixedWidth())
-            return getSizetConstant(map->getFixedRecordSize());
+        if (isFixedSizeRecord(record))
+            return getSizetConstant(getMinRecordSize(record));
 
         //call meta function mm.queryRecordSize(&row)
         StringBuffer metaInstance, temp;

ecl/hqlcpp/hqlcpp.ipp

@@ -543,7 +543,7 @@ class ColumnToOffsetMap;
 class RecordOffsetMap : public MapOf<IHqlExpression *, ColumnToOffsetMap>
 {
 public:
-    ColumnToOffsetMap * queryMapping(IHqlExpression * record, unsigned maxRecordSize);
+    ColumnToOffsetMap * queryMapping(IHqlExpression * record, unsigned maxRecordSize, bool isTarget);
 };
 
 class ExprExprMap : public MapOwnedToOwned<IHqlExpression, IHqlExpression>
@@ -589,6 +589,7 @@ struct HqlCppOptions
     unsigned            subgraphToRegenerate;
     unsigned            defaultPersistExpiry;
     unsigned            defaultExpiry;
+    unsigned            varFieldAccessorThreshold;
     int                 defaultNumPersistInstances;
     CompilerType        targetCompiler;
     DBZaction           divideByZeroAction;
@@ -1019,8 +1020,7 @@ public:
     BoundRow * bindCsvTableCursor(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq, bool translateVirtuals, IAtom * encoding);
     BoundRow * bindXmlTableCursor(BuildCtx & ctx, IHqlExpression * dataset, const char * name, node_operator side, IHqlExpression * selSeq, bool translateVirtuals);
     BoundRow * bindXmlTableCursor(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq, bool translateVirtuals);
-    BoundRow * createTableCursor(IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq);
-    BoundRow * createTableCursor(IHqlExpression * dataset, const char * name, bool isLinkCounted, node_operator side, IHqlExpression * selSeq);
+    BoundRow * createTableCursor(IHqlExpression * dataset, IHqlExpression * bound, bool useAccessorClass, node_operator side, IHqlExpression * selSeq);
     BoundRow * bindRow(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * bound);
     BoundRow * bindRow(BuildCtx & ctx, IHqlExpression * expr, const char * name);
     BoundRow * bindConstantRow(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & bound);
@@ -1057,7 +1057,7 @@ public:
     void getRecordSize(BuildCtx & ctx, IHqlExpression * dataset, CHqlBoundExpr & bound);
     BoundRow * resolveSelectorDataset(BuildCtx & ctx, IHqlExpression * dataset);
     BoundRow * resolveDatasetRequired(BuildCtx & ctx, IHqlExpression * expr);
-    ColumnToOffsetMap * queryRecordOffsetMap(IHqlExpression * record);
+    ColumnToOffsetMap * queryRecordOffsetMap(IHqlExpression * record, bool isTargetRow);
     IHqlExpression * queryRecord(BuildCtx & ctx, IHqlExpression * expr);
     RecordOffsetMap & queryRecordMap()              { return recordMap; }
     unsigned getDefaultMaxRecordSize()              { return options.maxRecordSize; }
@@ -1072,6 +1072,9 @@ public:
     void ensureRowPrefetcher(StringBuffer & prefetcherName, BuildCtx & ctx, IHqlExpression * record);
     IHqlExpression * createSerializer(BuildCtx & ctx, IHqlExpression * record, IAtom * format, IAtom * kind);
 
+    void buildRowAccessors();
+    void buildRowAccessor(ColumnToOffsetMap * map);
+
     AliasKind buildExprInCorrectContext(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, bool evaluateLocally);
     ParentExtract * createExtractBuilder(BuildCtx & ctx, PEtype type, IHqlExpression * graphId, IHqlExpression * expr, bool doDeclare);
     ParentExtract * createExtractBuilder(BuildCtx & ctx, PEtype type, IHqlExpression * graphId, GraphLocalisation localisation, bool doDeclare);
@@ -1791,8 +1794,8 @@ public:
     void buildRefFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, const char * name, IHqlExpression * dataset);
     void createAccessFunctions(StringBuffer & helperFunc, BuildCtx & declarectx, unsigned prio, const char * interfaceName, const char * object);
 
-    void beginNestedClass(BuildCtx & classctx, const char * member, const char * bases, const char * memberExtra = NULL, ParentExtract * extract = NULL);
-    void endNestedClass();
+    IHqlStmt * beginNestedClass(BuildCtx & classctx, const char * member, const char * bases, const char * memberExtra = NULL, ParentExtract * extract = NULL);
+    void endNestedClass(IHqlStmt * stmt);
 
     void buildEncryptHelper(BuildCtx & ctx, IHqlExpression * encryptAttr, const char * funcname = NULL);
     void buildFormatCrcFunction(BuildCtx & ctx, const char * name, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta);
@@ -1841,6 +1844,7 @@ protected:
     IHqlExpression * normalizeGlobalIfCondition(BuildCtx & ctx, IHqlExpression * expr);
     void substituteClusterSize(HqlExprArray & exprs);
     void throwCannotCast(ITypeInfo * from, ITypeInfo * to);
+    bool useRowAccessorClass(IHqlExpression * record, bool isTargetRow);
 
     void ensureSerialized(BuildCtx & ctx, const CHqlBoundTarget & variable);
 

ecl/hqlcpp/hqlcppds.cpp

@@ -684,8 +684,7 @@ BoundRow * HqlCppTranslator::ensureLinkCountedRow(BuildCtx & ctx, BoundRow * row
         return row;
 
     OwnedHqlExpr srcRow = createTranslated(row->queryBound());
-    OwnedHqlExpr tempRowExpr = declareLinkedRowExpr(ctx, row->queryRecord(), false);
-    Owned<BoundRow> tempRow = row->clone(tempRowExpr);
+    Owned<BoundRow> tempRow = declareLinkedRow(ctx, row->represents, false);
 
     OwnedHqlExpr source = getPointer(row->queryBound());
     BuildCtx subctx(ctx);

ecl/hqlcpp/hqlcset.cpp

@@ -1803,7 +1803,7 @@ void InlineDatasetBuilder::buildDeclare(BuildCtx & ctx)
 
 BoundRow * InlineDatasetBuilder::buildCreateRow(BuildCtx & ctx)
 {
-    Owned<BoundRow> cursor = translator.createTableCursor(dataset, cursorVar, no_self, NULL);
+    Owned<BoundRow> cursor = translator.createTableCursor(dataset, cursorVar, false, no_self, NULL);
     ctx.associate(*cursor);
     return cursor;
 }

ecl/hqlcpp/hqlecl.cpp

@@ -508,7 +508,7 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
 {
     cycle_t startCycles = get_cycles_now();
     unsigned numExtraFiles = translator.getNumExtraCppFiles();
-    bool isMultiFile = translator.spanMultipleCppFiles() && (numExtraFiles != 0);
+    bool isMultiFile = translator.spanMultipleCppFiles();
     CompilerType targetCompiler = translator.queryOptions().targetCompiler;
 
     StringBuffer fullname;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -240,12 +240,13 @@ static IHqlExpression * createResultName(IHqlExpression * name)
 
 //---------------------------------------------------------------------------
 
-ColumnToOffsetMap * RecordOffsetMap::queryMapping(IHqlExpression * record, unsigned maxRecordSize)
+ColumnToOffsetMap * RecordOffsetMap::queryMapping(IHqlExpression * record, unsigned maxRecordSize, bool useAccessorClass)
 {
-    ColumnToOffsetMap * match = find(record);
+    OwnedHqlExpr key = useAccessorClass ? createAttribute(classAtom, LINK(record)) : LINK(record);
+    ColumnToOffsetMap * match = find(key);
     if (!match)
     {
-        match = new ColumnToOffsetMap(record, 1, maxRecordSize, false);
+        match = new ColumnToOffsetMap(key, record, ordinality(), 1, maxRecordSize, false, useAccessorClass);
         match->init(*this);
         addOwn(*match);
     }
@@ -2967,7 +2968,7 @@ void getMemberClassName(StringBuffer & className, const char * member)
     className.append((char)toupper(member[0])).append(member+1).append("Class");
 }
 
-void HqlCppTranslator::beginNestedClass(BuildCtx & ctx, const char * member, const char * bases, const char * memberExtra, ParentExtract * extract)
+IHqlStmt * HqlCppTranslator::beginNestedClass(BuildCtx & ctx, const char * member, const char * bases, const char * memberExtra, ParentExtract * extract)
 {
 //  ActivityInstance * activity = queryCurrentActivity(ctx);
 //  Owned<ParentExtract> nestedUse;
@@ -2983,7 +2984,8 @@ void HqlCppTranslator::beginNestedClass(BuildCtx & ctx, const char * member, con
         begin.append(" : public ").append(bases);
     end.append(" ").append(member).append(memberExtra).append(";");
 
-    ctx.addQuotedCompound(begin.str(), end.str());
+    IHqlStmt * stmt = ctx.addQuotedCompound(begin.str(), end.str());
+    stmt->setIncomplete(true);
 
     OwnedHqlExpr colocalName = createVariable("activity", makeVoidType());
     ActivityInstance * activity = queryCurrentActivity(ctx);
@@ -2994,10 +2996,12 @@ void HqlCppTranslator::beginNestedClass(BuildCtx & ctx, const char * member, con
         ctx.associateOwn(*nested);
         nested->initContext();
     }
+    return stmt;
 }
 
-void HqlCppTranslator::endNestedClass()
+void HqlCppTranslator::endNestedClass(IHqlStmt * stmt)
 {
+    stmt->setIncomplete(false);
 }
 
 void HqlCppTranslator::doBuildFunctionReturn(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * value)
@@ -3300,10 +3304,9 @@ void HqlCppTranslator::getRecordSize(BuildCtx & ctx, IHqlExpression * dataset, C
 
 unsigned HqlCppTranslator::getMaxRecordSize(IHqlExpression * record)
 {
-    ColumnToOffsetMap * map = queryRecordOffsetMap(record);
-    if (!map)
+    if (!record)
         return 0;
-    return map->getMaxSize();
+    return ::getMaxRecordSize(record, options.maxRecordSize);
 }
 
 unsigned HqlCppTranslator::getCsvMaxLength(IHqlExpression * csvAttr)
@@ -3325,8 +3328,7 @@ unsigned HqlCppTranslator::getCsvMaxLength(IHqlExpression * csvAttr)
 
 bool HqlCppTranslator::isFixedWidthDataset(IHqlExpression * dataset)
 {
-    IHqlExpression * record = dataset->queryRecord();
-    return queryRecordOffsetMap(record)->isFixedWidth();
+    return isFixedSizeRecord(dataset->queryRecord());
 }
 
 
@@ -3493,7 +3495,7 @@ bool HqlCppTranslator::buildMetaPrefetcherClass(BuildCtx & ctx, IHqlExpression *
         MemberFunction func(*this, prefetcher.startctx, "virtual void readAhead(IRowDeserializerSource & in)");
         OwnedHqlExpr helper = createVariable("in", makeBoolType());
 
-        ok = queryRecordOffsetMap(record)->buildReadAhead(*this, func.ctx, helper);
+        ok = queryRecordOffsetMap(record, false)->buildReadAhead(*this, func.ctx, helper);
     }
 
     if (ok)
@@ -3526,7 +3528,7 @@ IHqlExpression * HqlCppTranslator::getRtlFieldKey(IHqlExpression * expr, IHqlExp
         {
         case type_bitfield:
             {
-                ColumnToOffsetMap * map = queryRecordOffsetMap(rowRecord);
+                ColumnToOffsetMap * map = queryRecordOffsetMap(rowRecord, false);
                 AColumnInfo * root = map->queryRootColumn();
                 CBitfieldInfo * resolved = static_cast<CBitfieldInfo *>(root->lookupColumn(expr));
                 assertex(resolved);
@@ -4115,13 +4117,12 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
     BuildCtx metactx(declarectx);
 
     IHqlExpression * record = instance.queryRecord();
-    ColumnToOffsetMap * map = queryRecordOffsetMap(record);
 
     unsigned flags = MDFhasserialize;       // we always generate a serialize since 
     bool useTypeForXML = false;
     if (instance.isGrouped())
         flags |= MDFgrouped;
-    if (map)
+    if (record)
         flags |= MDFhasxml;
     if (record)
     {
@@ -4155,7 +4156,7 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
     {
         //Serialization classes need to be generated for all meta information - because they may be called by parent row classes
         //however, the CFixedOutputMetaData base class contains a default implementation - reducing the required code.
-        if (map && (!map->isFixedWidth() || (flags & MDFneedserializemask)))
+        if (record && (isVariableSizeRecord(record) || (flags & MDFneedserializemask)))
         {
             //Base class provides a default variable width implementation
             if (flags & MDFneedserializedisk)
@@ -4193,9 +4194,9 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
         }
 
         s.append("struct ").append(instance.metaName).append(" : public ");
-        if (!map)
+        if (!record)
             s.append("CActionOutputMetaData");
-        else if (map->isFixedWidth())
+        else if (isFixedSizeRecord(record))
             s.append("CFixedOutputMetaData");
         else
             s.append("CVariableOutputMetaData");
@@ -4204,26 +4205,21 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
         IHqlStmt * metaclass = metactx.addQuotedCompound(s, endText.str());
         metaclass->setIncomplete(true);
 
-        if (map)
+        if (record)
         {
-            if (map->isFixedWidth())
+            if (isFixedSizeRecord(record))
             {
-                unsigned fixedSize = map->getFixedRecordSize();
+                unsigned fixedSize = getMinRecordSize(record);
                 s.clear().append("inline ").append(instance.metaName).append("() : CFixedOutputMetaData(").append(fixedSize).append(") {}");
                 metactx.addQuoted(s);
             }
             else
             {
                 unsigned minSize = getMinRecordSize(record);
-                unsigned maxLength = map->getMaxSize();
+                unsigned maxLength = getMaxRecordSize(record);
                 if (maxLength < minSize)
                     reportError(queryLocation(record), ECODETEXT(HQLERR_MaximumSizeLessThanMinimum_XY), maxLength, minSize);
                     
-#ifdef _DEBUG
-                //Paranoia check to ensure the two methods agree.
-                unsigned calcMinSize = map->getTotalMinimumSize();
-                assertex(minSize == calcMinSize);
-#endif
                 //These use a CVariableOutputMetaData base class instead, and trade storage for number of virtuals
                 s.clear().append("inline ").append(instance.metaName).append("() : CVariableOutputMetaData(").append(minSize).append(") {}");
                 metactx.addQuoted(s);
@@ -4781,10 +4777,9 @@ IHqlExpression * HqlCppTranslator::convertBetweenCountAndSize(const CHqlBoundExp
         UNIMPLEMENTED;
     }
 
-    ColumnToOffsetMap * map = queryRecordOffsetMap(record);
-    if (map->isFixedWidth())
+    if (isFixedSizeRecord(record))
     {
-        unsigned fixedSize = map->getFixedRecordSize();
+        unsigned fixedSize = getMinRecordSize(record);
         if (fixedSize == 0)
             throwError(HQLERR_ZeroLengthIllegal);
         if (type->getTypeCode() == type_groupedtable)
@@ -5478,7 +5473,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
 void HqlCppTranslator::buildCompareClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, IHqlExpression * datasetLeft, IHqlExpression * datasetRight, IHqlExpression * selSeq)
 {
     BuildCtx classctx(ctx);
-    beginNestedClass(classctx, name, "ICompare");
+    IHqlStmt * classStmt = beginNestedClass(classctx, name, "ICompare");
 
     {
         MemberFunction func(*this, classctx, "virtual int docompare(const void * _left, const void * _right) const" OPTIMIZE_FUNCTION_ATTRIBUTE);
@@ -5494,7 +5489,7 @@ void HqlCppTranslator::buildCompareClass(BuildCtx & ctx, const char * name, IHql
             buildReturn(func.ctx, orderExpr);
     }
 
-    endNestedClass();
+    endNestedClass(classStmt);
 }
 
 
@@ -5526,7 +5521,7 @@ void HqlCppTranslator::buildCompareMember(BuildCtx & ctx, const char * name, IHq
 void HqlCppTranslator::buildCompareEqClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, IHqlExpression * datasetLeft, IHqlExpression * datasetRight, IHqlExpression * selSeq)
 {
     BuildCtx classctx(ctx);
-    beginNestedClass(classctx, name, "ICompareEq");
+    IHqlStmt * classStmt = beginNestedClass(classctx, name, "ICompareEq");
 
     {
         MemberFunction func(*this, classctx, "virtual bool match(const void * _left, const void * _right) const");
@@ -5542,7 +5537,7 @@ void HqlCppTranslator::buildCompareEqClass(BuildCtx & ctx, const char * name, IH
             buildReturn(func.ctx, orderExpr);
     }
 
-    endNestedClass();
+    endNestedClass(classStmt);
 }
 
 
@@ -5560,7 +5555,7 @@ void HqlCppTranslator::buildCompareEqMemberLR(BuildCtx & ctx, const char * name,
 void HqlCppTranslator::buildNaryCompareClass(BuildCtx & ctx, const char * name, IHqlExpression * expr, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid)
 {
     BuildCtx classctx(ctx);
-    beginNestedClass(classctx, name, "INaryCompareEq");
+    IHqlStmt * classStmt = beginNestedClass(classctx, name, "INaryCompareEq");
 
     {
         MemberFunction func(*this, classctx, "virtual bool match(unsigned numRows, const void * * _rows) const");
@@ -5574,7 +5569,7 @@ void HqlCppTranslator::buildNaryCompareClass(BuildCtx & ctx, const char * name,
         buildReturn(func.ctx, expr);
     }
 
-    endNestedClass();
+    endNestedClass(classStmt);
 }
 
 
@@ -5625,7 +5620,7 @@ void HqlCppTranslator::buildHashClass(BuildCtx & ctx, const char * name, IHqlExp
     ctx.addQuoted(s);
 
     BuildCtx classctx(ctx);
-    beginNestedClass(classctx, name, "IHash");
+    IHqlStmt * classStmt = beginNestedClass(classctx, name, "IHash");
 
     {
         MemberFunction hashFunc(*this, classctx, "virtual unsigned hash(const void * _self)");
@@ -5636,14 +5631,14 @@ void HqlCppTranslator::buildHashClass(BuildCtx & ctx, const char * name, IHqlExp
         buildReturn(hashFunc.ctx, orderExpr, returnType);
     }
 
-    endNestedClass();
+    endNestedClass(classStmt);
 }
 
 
 void HqlCppTranslator::buildCompareClass(BuildCtx & ctx, const char * name, IHqlExpression * sortList, const DatasetReference & dataset)
 {
     BuildCtx comparectx(ctx);
-    beginNestedClass(comparectx, name, "ICompare");
+    IHqlStmt * classStmt = beginNestedClass(comparectx, name, "ICompare");
 
     {
         MemberFunction func(*this, comparectx, "virtual int docompare(const void * _left, const void * _right) const" OPTIMIZE_FUNCTION_ATTRIBUTE);
@@ -5654,7 +5649,7 @@ void HqlCppTranslator::buildCompareClass(BuildCtx & ctx, const char * name, IHql
         buildReturnOrder(func.ctx, sortList, dataset);
     }
 
-    endNestedClass();
+    endNestedClass(classStmt);
 }
 
 
@@ -5680,11 +5675,9 @@ void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, StringBu
         appendUniqueId(lookupHelperName.append("lu"), getConsistentUID(record));
 
         BuildCtx classctx(declarectx);
-        //I suspect all the priorities should be killed.  This is here because you can have type info constructors accessing the
-        //dictionary hash functions.
-        classctx.setNextPriority(HashFunctionPrio);
+        classctx.setNextPriority(TypeInfoPrio);
 
-        beginNestedClass(classctx, lookupHelperName, "IHThorHashLookupInfo");
+        IHqlStmt * classStmt = beginNestedClass(classctx, lookupHelperName, "IHThorHashLookupInfo");
         OwnedHqlExpr searchRecord = getDictionarySearchRecord(record);
         OwnedHqlExpr keyRecord = getDictionaryKeyRecord(record);
 
@@ -5722,7 +5715,7 @@ void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, StringBu
 
         buildCompareMemberLR(classctx, "CompareLookup", compare, source, dict, seq);
         buildCompareMember(classctx, "Compare", keyedDictList, dictRef);
-        endNestedClass();
+        endNestedClass(classStmt);
 
         if (queryOptions().spanMultipleCpp)
         {
@@ -5929,6 +5922,8 @@ bool HqlCppTranslator::buildCode(HqlQueryContext & query, const char * embeddedL
             wu()->setDebugValue("__Calculated__Complexity__", complexityText, true);
             noteFinishedTiming("compile:calculate complexity", startCycles);
         }
+
+        buildRowAccessors();
     }
 
     ::Release(outputLibrary);
@@ -7060,7 +7055,11 @@ void HqlCppTranslator::buildRecordSerializeExtract(BuildCtx & ctx, IHqlExpressio
 
 BoundRow * HqlCppTranslator::bindTableCursor(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq)
 {
-    BoundRow * cursor = createTableCursor(dataset, bound, side, selSeq);
+    IHqlExpression * record = dataset->queryRecord();
+    bool useRowAccessor = useRowAccessorClass(record, side == no_self);
+    BoundRow * cursor = createTableCursor(dataset, bound, useRowAccessor, side, selSeq);
+    if (useRowAccessor)
+        cursor->prepareAccessor(*this, ctx);
     ctx.associateOwn(*cursor);
     return cursor;
 }
@@ -7083,30 +7082,23 @@ BoundRow * HqlCppTranslator::rebindTableCursor(BuildCtx & ctx, IHqlExpression *
 }
 
 
-BoundRow * HqlCppTranslator::createTableCursor(IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq)
+BoundRow * HqlCppTranslator::createTableCursor(IHqlExpression * dataset, IHqlExpression * bound, bool useAccessorClass, node_operator side, IHqlExpression * selSeq)
 {
-    return new BoundRow(dataset, bound, queryRecordOffsetMap(dataset->queryRecord()), side, selSeq);
+    return new BoundRow(dataset, bound, NULL, queryRecordOffsetMap(dataset->queryRecord(), useAccessorClass), side, selSeq);
 }
 
 BoundRow * HqlCppTranslator::recreateTableCursor(IHqlExpression * dataset, BoundRow * row, node_operator side, IHqlExpression * selSeq)
 {
-    return new BoundRow(row, dataset, side, selSeq);
-}
+    ColumnToOffsetMap * columnMap = queryRecordOffsetMap(row->queryRecord(), false);
+    return new BoundRow(dataset, row->queryBound(), nullptr, columnMap, side, selSeq);
 
-BoundRow * HqlCppTranslator::createTableCursor(IHqlExpression * dataset, const char * name, bool isLinkCounted, node_operator side, IHqlExpression * selSeq)
-{
-    Owned<ITypeInfo> type = makeRowReferenceType(NULL);
-    if (isLinkCounted)
-        type.setown(makeAttributeModifier(type.getClear(), getLinkCountedAttr()));
-    Owned<IHqlExpression> bound = createVariable(name, type.getClear());
-    return createTableCursor(dataset, bound, side, selSeq);
 }
 
 BoundRow * HqlCppTranslator::bindXmlTableCursor(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * bound, node_operator side, IHqlExpression * selSeq, bool translateVirtuals)
 {
     Owned<ColumnToOffsetMap> xmlMap = new XmlColumnToOffsetMap(dataset->queryRecord(), getDefaultMaxRecordSize(), translateVirtuals);
     xmlMap->init(recordMap);
-    BoundRow * cursor = new BoundRow(dataset, bound, xmlMap, side, selSeq);
+    BoundRow * cursor = new BoundRow(dataset, bound, NULL, xmlMap, side, selSeq);
     ctx.associateOwn(*cursor);
     return cursor;
 }
@@ -7122,7 +7114,7 @@ BoundRow * HqlCppTranslator::bindCsvTableCursor(BuildCtx & ctx, IHqlExpression *
 {
     Owned<ColumnToOffsetMap> csvMap = new CsvColumnToOffsetMap(dataset->queryRecord(), getDefaultMaxRecordSize(), translateVirtuals, encoding);
     csvMap->init(recordMap);
-    BoundRow * cursor = new BoundRow(dataset, bound, csvMap, side, selSeq);
+    BoundRow * cursor = new BoundRow(dataset, bound, NULL, csvMap, side, selSeq);
     ctx.associateOwn(*cursor);
     return cursor;
 }
@@ -7177,7 +7169,9 @@ BoundRow * HqlCppTranslator::bindTableCursorOrRow(BuildCtx & ctx, IHqlExpression
 
 BoundRow * HqlCppTranslator::createBoundRow(IHqlExpression * dataset, IHqlExpression * bound)
 {
-    return new BoundRow(dataset->queryBody(), bound, queryRecordOffsetMap(dataset->queryRecord()));
+    bool useAccessor = false;
+    IHqlExpression * accessor = NULL;
+    return new BoundRow(dataset->queryBody(), bound, accessor, queryRecordOffsetMap(dataset->queryRecord(), (accessor != NULL)));
 }
 
 BoundRow * HqlCppTranslator::bindSelectorAsSelf(BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * expr)
@@ -7651,8 +7645,7 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
                 else
                 {
                     IHqlExpression * record = ::queryRecord(type);
-                    ColumnToOffsetMap * map = queryRecordOffsetMap(record);
-                    length.setown(getSizetConstant(map->getMaxSize()));
+                    length.setown(getSizetConstant(getMaxRecordSize(record)));
                 }
                 break;
             }
@@ -8018,11 +8011,10 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             case type_row:
                 {
                     OwnedHqlExpr record = getSerializedForm(child->queryRecord(), diskAtom);
-                    ColumnToOffsetMap * map = queryRecordOffsetMap(record);
-                    if (map->isFixedWidth())
-                        size = map->getFixedRecordSize();
+                    if (isFixedSizeRecord(record))
+                        size = getMinRecordSize(record);
                     else
-                        size = map->getMaxSize();
+                        size = getMaxRecordSize(record);
                 }
                 break;
             case type_alien:
@@ -8137,10 +8129,9 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
                 {
                     e->Release();
                     OwnedHqlExpr record = getSerializedForm(child->queryRecord(), diskAtom);
-                    ColumnToOffsetMap * map = queryRecordOffsetMap(record);
-                    if (map->isFixedWidth())
+                    if (isFixedSizeRecord(record))
                     {
-                        tgt.expr.setown(getSizetConstant(map->getFixedRecordSize()));
+                        tgt.expr.setown(getSizetConstant(getMinRecordSize(record)));
                         return;
                     }
                     throwError(HQLERR_CannotDetermineSizeVar);
@@ -9698,7 +9689,7 @@ void HqlCppTranslator::buildCsvParameters(BuildCtx & subctx, IHqlExpression * cs
 
     BuildCtx classctx(subctx);
     StringBuffer s;
-    beginNestedClass(classctx, "csv", "ICsvParameters");
+    IHqlStmt * classStmt = beginNestedClass(classctx, "csv", "ICsvParameters");
 
     doBuildBoolFunction(classctx, "queryEBCDIC", queryAttribute(ebcdicAtom, attrs)!=NULL);
 
@@ -9793,7 +9784,7 @@ void HqlCppTranslator::buildCsvParameters(BuildCtx & subctx, IHqlExpression * cs
 
     doBuildUnsignedFunction(classctx, "getFlags", flags.str()+1);
 
-    endNestedClass();
+    endNestedClass(classStmt);
 
     subctx.addQuotedLiteral("virtual ICsvParameters * queryCsvParameters() { return &csv; }");
 }
@@ -11695,7 +11686,7 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
         ctx.addQuoted(s);
 
         BuildCtx classctx(nestedctx);
-        beginNestedClass(classctx, compareName.str(), "ICompare");
+        IHqlStmt * classStmt = beginNestedClass(classctx, compareName.str(), "ICompare");
 
         {
             MemberFunction func(*this, classctx, "virtual int docompare(const void * _left, const void * _right) const" OPTIMIZE_FUNCTION_ATTRIBUTE);
@@ -11707,7 +11698,7 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
             buildReturnOrder(func.ctx, groupOrder, dataset);
         }
 
-        endNestedClass();
+        endNestedClass(classStmt);
     }
 }
 
@@ -11774,8 +11765,8 @@ void HqlCppTranslator::generateSerializeFunction(BuildCtx & ctx, const char * fu
     func.ctx.addQuotedLiteral("const unsigned char * src = (const unsigned char *) _src;");
 
     OwnedHqlExpr selSeq = createDummySelectorSequence();
-    BoundRow * tgtCursor = bindSelf(ctx, tgtDataset.queryDataset(), "crSelf");
-    BoundRow * srcCursor = bindTableCursor(ctx, srcDataset.queryDataset(), "src", no_left, selSeq);
+    BoundRow * tgtCursor = bindSelf(func.ctx, tgtDataset.queryDataset(), "crSelf");
+    BoundRow * srcCursor = bindTableCursor(func.ctx, srcDataset.queryDataset(), "src", no_left, selSeq);
 
     IHqlExpression * leftSelect = srcCursor->querySelector();
     IHqlExpression * selfSelect = tgtCursor->querySelector();
@@ -11877,7 +11868,7 @@ void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator
     memberName.append("serializer").append(sideText);
 
     BuildCtx classctx(nestedctx);
-    beginNestedClass(classctx, memberName, "ISortKeySerializer");
+    IHqlStmt * classStmt = beginNestedClass(classctx, memberName, "ISortKeySerializer");
 
     DatasetReference keyActiveRef(keyInfo.keyDataset, no_activetable, NULL);
     OwnedHqlExpr keyOrder = createValueSafe(no_sortlist, makeSortListType(NULL), keyInfo.keyCompares);
@@ -11889,7 +11880,7 @@ void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator
     buildCompareMember(classctx, "CompareKey", keyOrder, keyActiveRef);
     doCompareLeftRight(classctx, "CompareKeyRow", keyActiveRef, dataset, keyInfo.keyCompares, keyInfo.filteredSorts);
 
-    endNestedClass();
+    endNestedClass(classStmt);
 
     s.clear().append("virtual ISortKeySerializer * querySerialize").append(sideText).append("() { return &").append(memberName).append("; }");
     nestedctx.addQuoted(s);
@@ -14130,8 +14121,8 @@ void HqlCppTranslator::buildDedupSerializeFunction(BuildCtx & ctx, const char *
     ensureRowAllocated(func.ctx, "crSelf");
     func.ctx.addQuotedLiteral("const unsigned char * src = (const unsigned char *) _src;");
 
-    BoundRow * tgtCursor = bindSelf(ctx, tgtDataset, "crSelf");
-    BoundRow * srcCursor = bindTableCursor(ctx, srcDataset, "src", no_left, selSeq);
+    BoundRow * tgtCursor = bindSelf(func.ctx, tgtDataset, "crSelf");
+    BoundRow * srcCursor = bindTableCursor(func.ctx, srcDataset, "src", no_left, selSeq);
     ForEachItemIn(idx2, srcValues)
     {
         Owned<IHqlExpression> self = tgtCursor->bindToRow(&tgtValues.item(idx2), queryActiveTableSelector());
@@ -14807,19 +14798,20 @@ ABoundActivity * HqlCppTranslator::doBuildActivityNormalizeChild(BuildCtx & ctx,
         getMemberClassName(className, memberName.str());
 
         ExpressionFormat format;
+        IHqlStmt * classStmt = nullptr;
         if (streamed)
         {
-            beginNestedClass(iterclassctx, memberName, "CNormalizeStreamedChildIterator");
+            classStmt = beginNestedClass(iterclassctx, memberName, "CNormalizeStreamedChildIterator");
             format = FormatStreamedDataset;
         }
         else if (outOfLine)
         {
-            beginNestedClass(iterclassctx, memberName, "CNormalizeLinkedChildIterator");
+            classStmt = beginNestedClass(iterclassctx, memberName, "CNormalizeLinkedChildIterator");
             format = FormatLinkedDataset;
         }
         else
         {
-            beginNestedClass(iterclassctx, memberName, "CNormalizeChildIterator");
+            classStmt = beginNestedClass(iterclassctx, memberName, "CNormalizeChildIterator");
             format = FormatBlockedDataset;
 
             MetaInstance childmeta(*this, childDataset->queryRecord(), isGrouped(childDataset));
@@ -14892,7 +14884,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityNormalizeChild(BuildCtx & ctx,
                 func.ctx.addQuoted(s);
         }
 
-        endNestedClass();
+        endNestedClass(classStmt);
 
         s.clear().append("INormalizeChildIterator * queryIterator() { return &").append(memberName).append("; }");
         instance->startctx.addQuoted(s);
@@ -18771,22 +18763,35 @@ void HqlCppTranslator::buildConnectOrders(BuildCtx & ctx, ABoundActivity * slave
 }
 
 
-ColumnToOffsetMap * HqlCppTranslator::queryRecordOffsetMap(IHqlExpression * record)
+bool HqlCppTranslator::useRowAccessorClass(IHqlExpression * record, bool isTargetRow)
+{
+    if (isTargetRow)
+        return false;
+    if (isFixedRecordSize(record))
+        return false;
+    if (!canCreateRtlTypeInfo(record))
+        return false;
+    return getVarSizeFieldCount(record, true) >= options.varFieldAccessorThreshold;
+}
+
+
+ColumnToOffsetMap * HqlCppTranslator::queryRecordOffsetMap(IHqlExpression * record, bool useAccessorClass)
 {
     if (record)
-        return recordMap.queryMapping(record, options.maxRecordSize);
+        return recordMap.queryMapping(record, options.maxRecordSize, useAccessorClass);
     return NULL;
 }
 
 unsigned HqlCppTranslator::getFixedRecordSize(IHqlExpression * record)
 {
-    return queryRecordOffsetMap(record)->getFixedRecordSize();
+    assertex(isFixedSizeRecord(record));
+    return getMinRecordSize(record);
 }
 
 
 bool HqlCppTranslator::isFixedRecordSize(IHqlExpression * record)
 {
-    return queryRecordOffsetMap(record)->isFixedWidth();
+    return ::isFixedSizeRecord(record);
 }
 
 void HqlCppTranslator::buildReturnRecordSize(BuildCtx & ctx, BoundRow * cursor)
@@ -18798,7 +18803,38 @@ void HqlCppTranslator::buildReturnRecordSize(BuildCtx & ctx, BoundRow * cursor)
 
 bool HqlCppTranslator::recordContainsIfBlock(IHqlExpression * record)
 {
-    return queryRecordOffsetMap(record)->queryContainsIfBlock();
+    return ::containsIfBlock(record);
+}
+
+
+void HqlCppTranslator::buildRowAccessors()
+{
+    HashIterator iter(recordMap);
+    ForEach(iter)
+    {
+        ColumnToOffsetMap * map = static_cast<ColumnToOffsetMap *>(&iter.query());
+        buildRowAccessor(map);
+    }
+
+}
+
+void HqlCppTranslator::buildRowAccessor(ColumnToOffsetMap * map)
+{
+    if (!map->usesAccessor())
+        return;
+
+    const bool isRead = true;
+    IHqlExpression * record = map->queryRecord();
+    BuildCtx declarectx(*code, declareAtom);
+    BuildCtx ctx(declarectx);
+    OwnedHqlExpr search = createAttribute(accessorAtom, LINK(record), createConstant(isRead));
+    if (ctx.queryMatchExpr(search))
+        return;
+
+    StringBuffer accessorName;
+    map->buildAccessor(accessorName, *this, ctx, NULL);
+
+    declarectx.associateExpr(search, search);
 }
 
 //-- Code to transform the expressions ready for generating source code.

ecl/hqlcpp/hqlinline.cpp

@@ -1358,7 +1358,8 @@ void ParentExtract::gatherActiveRows(BuildCtx & ctx)
             else if (!cur.isBinary())
             {
                 //CSV and xml datasets need their elements serialized into the parent extract
-                newRow = new NonLocalIndirectRow(cur, NULL, childSerialization);
+                ColumnToOffsetMap * map = translator.queryRecordOffsetMap(cur.queryRecord(), false);
+                newRow = new NonLocalIndirectRow(cur, map, childSerialization);
             }
             else if (serialization)
             {
@@ -1382,7 +1383,10 @@ void ParentExtract::gatherActiveRows(BuildCtx & ctx)
                 else
                     expandedAlias.set(&matchAlias.item(match));
 
-                newRow = new BoundAliasRow(cur, NULL, expandedAlias);
+                //At the moment do not use offset classes in the child query, until we also serialize a pointeter to the
+                //offset class.  This would be much better once co-local extracts were implemented with c++ classes.
+                ColumnToOffsetMap * map = translator.queryRecordOffsetMap(cur.queryRecord(), false);
+                newRow = new BoundAliasRow(cur, map, expandedAlias);
                 newRow->setInherited(true);
             }
             else
@@ -1408,7 +1412,8 @@ void ParentExtract::gatherActiveRows(BuildCtx & ctx)
         ForEachItemInRev(i, activeRows)
         {
             BoundRow & cur = activeRows.item(i);
-            nonlocalBoundCursors.append(*new NonLocalIndirectRow(cur, NULL, childSerialization));
+            ColumnToOffsetMap * map = translator.queryRecordOffsetMap(cur.queryRecord(), false);
+            nonlocalBoundCursors.append(*new NonLocalIndirectRow(cur, map, childSerialization));
         }
     }
 }

ecl/hqlcpp/hqliter.cpp

@@ -215,7 +215,7 @@ void CompoundIteratorBuilder::bindParentCursors(BuildCtx & ctx, CursorArray & cu
         BoundRow & cur = cursors.item(i);
         //Very similar to code in the extract builder
         OwnedHqlExpr colocalBound = addMemberSelector(cur.queryBound(), colocal);
-        ctx.associateOwn(*cur.clone(colocalBound));
+        translator.bindTableCursor(ctx, cur.queryDataset(), colocalBound, cur.querySide(), cur.querySelSeq());
     }
 }
 
@@ -253,7 +253,7 @@ void CompoundIteratorBuilder::createSingleLevelIterator(StringBuffer & iterName,
     translator.getUniqueId(cursorName.append("row"));
     OwnedHqlExpr row = createVariable(cursorName, makeRowReferenceType(cur));
     declarectx.addDeclare(row);
-    cursors.append(*translator.createTableCursor(cur, row, no_none, NULL));
+    cursors.append(*translator.createTableCursor(cur, row, false, no_none, NULL));
 }
 
 void CompoundIteratorBuilder::createSingleIterator(StringBuffer & iterName, IHqlExpression * expr, CursorArray & cursors)
@@ -263,7 +263,7 @@ void CompoundIteratorBuilder::createSingleIterator(StringBuffer & iterName, IHql
 
     //MORE: Nested class/...
     BuildCtx classctx(nestedctx);
-    translator.beginNestedClass(classctx, iterName, "IRtlDatasetSimpleCursor", NULL, NULL);
+    IHqlStmt * classStmt = translator.beginNestedClass(classctx, iterName, "IRtlDatasetSimpleCursor", NULL, NULL);
     translator.queryEvalContext(classctx)->ensureHelpersExist();
 
     if (isArrayRowset(expr->queryType()))
@@ -322,7 +322,7 @@ void CompoundIteratorBuilder::createSingleIterator(StringBuffer & iterName, IHql
         nextctx.addQuotedLiteral("const byte * valid = checkValid(); if (valid) return valid;");
     }
 
-    translator.endNestedClass();
+    translator.endNestedClass(classStmt);
 }
 
 void CompoundIteratorBuilder::createRawFirstFunc(BuildCtx & ctx, IHqlExpression * expr, CursorArray & cursors)

ecl/hqlcpp/hqlnlp.cpp

@@ -220,7 +220,7 @@ void NlpParseContext::buildValidators(HqlCppTranslator & translator, BuildCtx &
     if (validators.ordinality())
     {
         BuildCtx helperctx(classctx);
-        translator.beginNestedClass(helperctx, "helper", "INlpHelper");
+        IHqlStmt * helperClassStmt = translator.beginNestedClass(helperctx, "helper", "INlpHelper");
 
         BuildCtx funcctx(helperctx);
         funcctx.addQuotedFunction("virtual IValidator * queryValidator(unsigned i)");
@@ -237,7 +237,7 @@ void NlpParseContext::buildValidators(HqlCppTranslator & translator, BuildCtx &
             ValidateKind kind = getValidateKind(validateExpr);
 
             BuildCtx validatorctx(helperctx);
-            translator.beginNestedClass(validatorctx, member, (kind != ValidateIsUnicode) ? "IStringValidator" : "IUnicodeValidator");
+            IHqlStmt * classStmt = translator.beginNestedClass(validatorctx, member, (kind != ValidateIsUnicode) ? "IStringValidator" : "IUnicodeValidator");
 
             {
                 MemberFunction func(translator, validatorctx);
@@ -264,7 +264,7 @@ void NlpParseContext::buildValidators(HqlCppTranslator & translator, BuildCtx &
                     validateExpr.setown(spotScalarCSE(validateExpr, NULL, translator.queryOptions().spotCseInIfDatasetConditions));
                 translator.buildReturn(func.ctx, validateExpr);
             }
-            translator.endNestedClass();
+            translator.endNestedClass(classStmt);
 
             StringBuffer s;
             s.append("case ").append(idx).append(": return &").append(member).append(";");
@@ -272,7 +272,7 @@ void NlpParseContext::buildValidators(HqlCppTranslator & translator, BuildCtx &
         }
         funcctx.addReturn(queryQuotedNullExpr());
 
-        translator.endNestedClass();
+        translator.endNestedClass(helperClassStmt);
         classctx.addQuotedLiteral("virtual INlpHelper * queryHelper() { return &helper; }");
     }
 }

ecl/hqlcpp/hqlsource.cpp

@@ -1186,7 +1186,8 @@ void SourceBuilder::buildTransformBody(BuildCtx & transformCtx, IHqlExpression *
         {
             //NOTE: The source is not link counted - it comes from a prefetched row, and does not include any virtual file position field.
             OwnedHqlExpr boundSrc = createVariable("left", makeRowReferenceType(physicalRecord));
-            transformCtx.associateOwn(*new BoundRow(tableExpr->queryNormalizedSelector(), boundSrc, translator.queryRecordOffsetMap(physicalRecord), no_none, NULL));
+            IHqlExpression * accessor = NULL;
+            transformCtx.associateOwn(*new BoundRow(tableExpr->queryNormalizedSelector(), boundSrc, accessor, translator.queryRecordOffsetMap(physicalRecord, (accessor != NULL)), no_none, NULL));
         }
     }
 
@@ -2147,7 +2148,7 @@ void SourceBuilder::buildGroupAggregateHashHelper(ParentExtract * extractBuilder
     instance->classctx.addQuotedLiteral("virtual IHash * queryHash() { return &hash; }");
 
     BuildCtx classctx(instance->nestedctx);
-    translator.beginNestedClass(classctx, "hash", "IHash", NULL, extractBuilder);
+    IHqlStmt * classStmt = translator.beginNestedClass(classctx, "hash", "IHash", NULL, extractBuilder);
 
     {
         MemberFunction func(translator, classctx, "virtual unsigned hash(const void * _self)");
@@ -2157,7 +2158,7 @@ void SourceBuilder::buildGroupAggregateHashHelper(ParentExtract * extractBuilder
         translator.buildReturn(func.ctx, hash);
     }
 
-    translator.endNestedClass();
+    translator.endNestedClass(classStmt);
 }
 
 void SourceBuilder::buildGroupAggregateCompareHelper(ParentExtract * extractBuilder, IHqlExpression * aggregate, HqlExprArray & recordFields, HqlExprArray & aggregateFields)
@@ -2211,7 +2212,7 @@ void SourceBuilder::buildGroupAggregateCompareHelper(ParentExtract * extractBuil
     instance->classctx.addQuotedLiteral("virtual ICompare * queryCompareRowElement() { return &compareRowElement; }");
 
     BuildCtx classctx(instance->nestedctx);
-    translator.beginNestedClass(classctx, "compareRowElement", "ICompare", NULL, extractBuilder);
+    IHqlStmt * classStmt = translator.beginNestedClass(classctx, "compareRowElement", "ICompare", NULL, extractBuilder);
 
     {
         MemberFunction func(translator, classctx, "virtual int docompare(const void * _left, const void * _right) const");
@@ -2223,7 +2224,7 @@ void SourceBuilder::buildGroupAggregateCompareHelper(ParentExtract * extractBuil
         translator.doBuildReturnCompare(func.ctx, order, no_eq, false, false);
     }
 
-    translator.endNestedClass();
+    translator.endNestedClass(classStmt);
 }
 
 

ecl/hqlcpp/hqlstmt.hpp

@@ -159,7 +159,7 @@ public:
     void                        set(BuildCtx & _owner);
 
 public:
-    enum { ConPrio = 1, EarlyPrio =3000, NormalPrio = 5000, LatePrio = 7000, DesPrio = 9999, OutermostScopePrio };
+    enum { ConPrio = 1, NormalPrio = 5000, DesPrio = 9999, OutermostScopePrio };
 
 protected:
     HqlStmt *                   appendCompound(HqlCompoundStmt * next);

ecl/hqlcpp/hqltcppc.cpp

@@ -103,18 +103,35 @@ IHqlExpression * ensureType(IHqlExpression * expr, ITypeInfo * type)
     return expr;
 }
 
+static bool isVerySimpleLength(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_variable:
+    case no_constant:
+    case no_select:
+        return true;
+    }
+    return false;
+}
 bool isSimpleLength(IHqlExpression * expr)
 {
     switch (expr->getOperator())
     {
     case no_variable:
     case no_constant:
+    case no_select:
         return true;
     case no_add:
-        if ((expr->queryChild(0)->getOperator() == no_variable) &&
+        if (isVerySimpleLength(expr->queryChild(0)) &&
             (expr->queryChild(1)->getOperator() == no_constant))
             return true;
         break;
+    case no_sub:
+        if (isVerySimpleLength(expr->queryChild(0)))
+            if (isSimpleLength(expr->queryChild(1)))
+                    return true;
+        break;
     }
     return false;
 }
@@ -133,6 +150,17 @@ void ensureSimpleLength(HqlCppTranslator & translator, BuildCtx & ctx, CHqlBound
 
 //---------------------------------------------------------------------------
 
+static IHqlExpression * createSizeExpression(IHqlExpression * varSize, unsigned fixedSize)
+{
+    if (!varSize)
+        return getSizetConstant(fixedSize);
+
+    OwnedHqlExpr total = ensureType(LINK(varSize), sizetType);
+    if (fixedSize)
+        return adjustValue(total, (int)fixedSize);
+    return total.getClear();
+}
+
 void SizeStruct::add(const SizeStruct & other)
 {
     assertex(self == other.self);
@@ -171,15 +199,12 @@ void SizeStruct::buildSizeExpr(HqlCppTranslator & translator, BuildCtx & ctx, Bo
 
 void SizeStruct::forceToTemp(node_operator op, IHqlExpression * selector)
 {
-    if (isWorthCommoning())
-    {
-        varSize.setown(createValue(op, LINK(sizetType), LINK(selector)));
-        fixedSize = 0;
-    }
+    varSize.setown(createValue(op, LINK(sizetType), LINK(selector)));
+    fixedSize = 0;
 }
 
 
-IHqlExpression * SizeStruct::getSizeExpr(BoundRow * row)
+IHqlExpression * SizeStruct::getSizeExpr(BoundRow * row) const
 {
 #if 0
     IHqlExpression * bound = row->queryDataset();
@@ -187,23 +212,26 @@ IHqlExpression * SizeStruct::getSizeExpr(BoundRow * row)
         return createTranslated(bound->queryChild(1));
 #endif
 
-    assertex(self != NULL);
-    OwnedHqlExpr mapped = normalizeAdditions(varSize);
-    OwnedHqlExpr total = row->bindToRow(mapped, self);
-    if (!total)
-        return getSizetConstant(fixedSize);
-
-    total.setown(ensureType(LINK(total), sizetType));
-    if (fixedSize)
-        return adjustValue(total, (int)fixedSize);
-    return total.getClear();
+    OwnedHqlExpr total;
+    if (row)
+    {
+        assertex(self != NULL);
+        OwnedHqlExpr mapped = normalizeAdditions(varSize);
+        total.setown(row->bindToRow(mapped, self));
+    }
+    else
+        total.set(varSize);
+    return createSizeExpression(total, fixedSize);
 }
 
 
 bool SizeStruct::isWorthCommoning() const
 {
-    if (varSize && varSize->getOperator() == no_add && varSize->queryChild(0)->getOperator() == no_add)
-        return true;
+    if (varSize && varSize->getOperator() == no_add)
+    {
+        if (varSize->queryChild(0)->getOperator() == no_add)
+            return true;
+    }
     return false;
 }
 
@@ -219,6 +247,7 @@ CMemberInfo::CMemberInfo(CContainerInfo * _container, CMemberInfo * _prior, IHql
         column.setown(createRow(no_null, LINK(column)));
     hasVarOffset = false;
     isOffsetCached = false;
+    seq = 0;
 }
 
 void CMemberInfo::addVariableSize(size32_t varMinSize, SizeStruct & size)
@@ -262,6 +291,13 @@ void CMemberInfo::calcCachedSize(const SizeStruct & offset, SizeStruct & sizeSel
 }
 
 
+void CMemberInfo::getOffsets(SizeStruct & offset, SizeStruct & accessorOffset) const
+{
+    offset.set(cachedOffset);
+    accessorOffset.set(cachedAccessorOffset);
+}
+
+
 bool CMemberInfo::checkCompatibleIfBlock(HqlExprCopyArray & conditions)
 {
     return false;
@@ -302,13 +338,31 @@ void CContainerInfo::calcAllCachedOffsets()
         SizeStruct offset(self);
         SizeStruct size(self);
         calcCachedOffsets(offset, size);
+
+        if (usesAccessClass())
+        {
+            SizeStruct finalAccessorOffset(self);
+            OwnedHqlExpr sizeSelf = createValue(no_sizeof, LINK(sizetType), getRelativeSelf());
+            if (bindOffsetsFromClass(finalAccessorOffset, false))
+            {
+                StringBuffer name;
+                name.append("off[").append(nextSeq()).append("]");
+                OwnedHqlExpr newOffset = createVariable(name, LINK(sizetType));
+                finalAccessorOffset.set(0, newOffset);
+            }
+            accessorSize.set(finalAccessorOffset);
+            cachedSize.set(0, sizeSelf);
+
+            SizeStruct tempOffset;
+            bindSizesFromOffsets(tempOffset, cachedSize);
+        }
     }
 }
 
 void CContainerInfo::calcCachedChildrenOffsets(const SizeStruct & startOffset, SizeStruct & sizeSelf)
 {
     //Optimize one special case of ifblocks.
-    //Sometimes you have a header with some fields, followed by a set of mutually exlusive ifblocks.
+    //Sometimes you have a header with some fields, followed by a set of mutually exclusive ifblocks.
     //Spot any trailing mutually exclusive ifblocks and don't update
     HqlExprCopyArray conditions;
     unsigned maxOffsetUpdate = children.ordinality();
@@ -363,6 +417,70 @@ void CContainerInfo::calcCachedSize(const SizeStruct & offset, SizeStruct & size
         addVariableSize(cachedSize.getMinimumSize(), sizeSelf);
 }
 
+bool CMemberInfo::bindOffsetsFromClass(SizeStruct & accessorOffset, bool prevVariableSize)
+{
+    if (prevVariableSize)
+    {
+        seq = container->nextSeq();
+        StringBuffer name;
+        name.append("off[").append(seq).append("]");
+        OwnedHqlExpr newOffset = createVariable(name, LINK(sizetType));
+        cachedAccessorOffset.set(0, newOffset);
+    }
+    else
+        cachedAccessorOffset.set(accessorOffset);
+
+    if (!cachedAccessorOffset.isFixedSize())
+    {
+        Owned<IHqlExpression> child = createSelectorExpr();
+        cachedOffset.forceToTemp(no_offsetof, child);
+    }
+    else
+        assertex(!cachedOffset.queryVarSize() || cachedOffset.queryVarSize()->getOperator() != no_add);
+
+    accessorOffset.set(cachedAccessorOffset);
+    accessorOffset.addFixed(cachedSize.getFixedSize());
+    return !cachedSize.isFixedSize();
+}
+
+bool CContainerInfo::bindOffsetsFromClass(SizeStruct & accessorOffset, bool prevVariableSize)
+{
+    //MORE: ifblocks need further work if the offsets are not recalculated for trailing ifblocks
+    ForEachItemIn(idx, children)
+    {
+        CMemberInfo & cur = children.item(idx);
+        bool thisVariableSize = cur.bindOffsetsFromClass(accessorOffset, prevVariableSize);
+        if (idx == 0)
+            cur.getOffsets(cachedOffset, cachedAccessorOffset);
+
+        prevVariableSize = thisVariableSize;
+    }
+
+    return prevVariableSize;
+}
+
+void CMemberInfo::bindSizesFromOffsets(SizeStruct & thisOffset, const SizeStruct & nextOffset)
+{
+    if (!cachedSize.isFixedSize())
+    {
+        assertex(nextOffset.queryVarSize()->getOperator() != no_add);
+        OwnedHqlExpr sub = createValue(no_sub, LINK(sizetType), nextOffset.getSizeExpr(NULL), cachedOffset.getSizeExpr(NULL));
+        cachedSize.set(0, sub);
+    }
+    thisOffset.set(cachedOffset);
+}
+
+void CContainerInfo::bindSizesFromOffsets(SizeStruct & thisOffset, const SizeStruct & nextOffset)
+{
+    SizeStruct curOffset(nextOffset);
+    ForEachItemInRev(idx, children)
+    {
+        CMemberInfo & cur = children.item(idx);
+        cur.bindSizesFromOffsets(curOffset, curOffset);
+    }
+    CMemberInfo::bindSizesFromOffsets(thisOffset, nextOffset);
+}
+
 unsigned CContainerInfo::getTotalFixedSize()
 {
     if (!isOffsetCached)
@@ -475,10 +593,24 @@ void CMemberInfo::buildConditionFilter(HqlCppTranslator & translator, BuildCtx &
 
 void CMemberInfo::buildOffset(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound)
 {
-    SizeStruct totalSize;
-    gatherOffset(totalSize, selector->queryExpr());     //this
+    if (!cachedAccessorOffset.isEmpty())
+    {
+        OwnedHqlExpr value;
+        if (cachedAccessorOffset.queryVarSize())
+        {
+            IHqlExpression * accessor = selector->queryRootRow()->ensureAccessor(translator, ctx);
+            assertex(accessor);
+            value.setown(createValue(no_select, LINK(sizetType), LINK(accessor), LINK(cachedAccessorOffset.queryVarSize())));
+        }
+        bound.expr.setown(createSizeExpression(value, cachedAccessorOffset.getFixedSize()));
+    }
+    else
+    {
+        SizeStruct totalSize;
+        gatherOffset(totalSize, selector->queryExpr());     //this
 
-    totalSize.buildSizeExpr(translator, ctx, selector->queryRootRow(), bound);
+        totalSize.buildSizeExpr(translator, ctx, selector->queryRootRow(), bound);
+    }
 }
 
 void callDeserializeGetN(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper, IHqlExpression * boundSize, IHqlExpression * address)
@@ -954,8 +1086,18 @@ void CContainerInfo::buildSizeOf(HqlCppTranslator & translator, BuildCtx & ctx,
 {
     if (!isOffsetCached)
         calcAllCachedOffsets();
-
-    cachedSize.buildSizeExpr(translator, ctx, selector->queryRootRow(), bound);
+    if (container && !usesAccessClass())
+        usesAccessClass();
+    if (!container && usesAccessClass())
+    {
+        IHqlExpression * accessor = selector->queryRootRow()->ensureAccessor(translator, ctx);
+        assertex(accessor);
+        if (accessorSize.queryVarSize())
+            bound.expr.setown(createValue(no_select, LINK(sizetType), LINK(accessor), LINK(accessorSize.queryVarSize())));
+        bound.expr.setown(createSizeExpression(bound.expr, accessorSize.getFixedSize()));
+    }
+    else
+        cachedSize.buildSizeExpr(translator, ctx, selector->queryRootRow(), bound);
 }
 
 
@@ -1089,6 +1231,13 @@ void CContainerInfo::getContainerXPath(StringBuffer & out)
     }
 }
 
+unsigned CContainerInfo::nextSeq()
+{
+    if (container)
+        return container->nextSeq();
+    return ++seq;
+}
+
 
 bool CContainerInfo::isConditional()
 {
@@ -1108,6 +1257,7 @@ void CContainerInfo::registerChild(CMemberInfo * child)
 CRecordInfo::CRecordInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column) :
     CContainerInfo(_container, _prior, _column)
 {
+    useAccessClass = false;
 }
 
 void CRecordInfo::buildAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target)     
@@ -2915,7 +3065,8 @@ void CXmlColumnInfo::setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IR
 
 inline int doAlign(unsigned value, unsigned align) { return (value + align-1) & ~(align-1); }
 
-ColumnToOffsetMap::ColumnToOffsetMap(IHqlExpression * _record, unsigned _packing, unsigned _maxRecordSize, bool _translateVirtuals) : root(NULL, NULL, _record) 
+ColumnToOffsetMap::ColumnToOffsetMap(IHqlExpression * _key, IHqlExpression * _record, unsigned _id, unsigned _packing, unsigned _maxRecordSize, bool _translateVirtuals, bool _useAccessClass)
+: root(NULL, NULL, _record), key(_key), id(_id)
 { 
     record = _record;
     prior = NULL;
@@ -2928,6 +3079,8 @@ ColumnToOffsetMap::ColumnToOffsetMap(IHqlExpression * _record, unsigned _packing
     cachedDefaultMaxSizeUsed = false;
     cachedMaxSize = UNKNOWN_LENGTH;
     root.setOffset(false);
+    if (_useAccessClass)
+        root.setUseAccessClass();
 }
 
 void ColumnToOffsetMap::init(RecordOffsetMap & map)
@@ -3006,6 +3159,36 @@ bool ColumnToOffsetMap::buildReadAhead(HqlCppTranslator & translator, BuildCtx &
 }
 
 
+void ColumnToOffsetMap::buildAccessor(StringBuffer & accessorName, HqlCppTranslator & translator, BuildCtx & declarectx, IHqlExpression * selector)
+{
+    StringBuffer typeName;
+    translator.buildRtlType(typeName, record->queryType(), 0);
+
+    BuildCtx ctx(declarectx);
+    ctx.setNextPriority(TypeInfoPrio);
+    ctx.addGroup();
+
+    StringBuffer s;
+    s.clear().append("RtlRecord rec").append(queryId()).append("(").append(typeName).append(",true);");
+    ctx.addQuoted(s);
+
+    if (translator.queryOptions().spanMultipleCpp)
+    {
+        ctx.set(mainprototypesAtom);
+        ctx.addQuoted(s.clear().append("extern RtlRecord rec").append(queryId()).append(";"));
+    }
+
+    BuildCtx classctx(ctx);
+    unsigned numVarOffsets = getVarSizeFieldCount(record, true);
+    accessorName.append("access").append(queryId());
+    s.clear().append("struct ").append(accessorName).append(" : public RtlStaticRow<").append(numVarOffsets).append(">");
+    classctx.addQuotedCompound(s.str(), ";");
+
+    s.clear().append(accessorName).append("(const void * _row) : RtlStaticRow<").append(numVarOffsets).append(">(rec").append(queryId()).append(", _row) {}");
+    classctx.addQuoted(s.str());
+}
+
+
 CMemberInfo * ColumnToOffsetMap::createColumn(CContainerInfo * container, IHqlExpression * column, RecordOffsetMap & map)
 {
     ITypeInfo * type = column->queryType();
@@ -3197,7 +3380,7 @@ CMemberInfo * ColumnToOffsetMap::expandRecord(IHqlExpression * record, CContaine
 }
 
 
-DynamicColumnToOffsetMap::DynamicColumnToOffsetMap(unsigned _maxRecordSize) : ColumnToOffsetMap(queryNullRecord(), 0, _maxRecordSize, false)
+DynamicColumnToOffsetMap::DynamicColumnToOffsetMap(unsigned _maxRecordSize) : ColumnToOffsetMap(NULL, queryNullRecord(), 0, 0, _maxRecordSize, false, false)
 {
     root.setDynamic();
 }
@@ -3264,7 +3447,8 @@ bool canReadFromCsv(IHqlExpression * record)
 }
 
 
-CsvColumnToOffsetMap::CsvColumnToOffsetMap(IHqlExpression * _record, unsigned _maxRecordSize, bool _translateVirtuals, IAtom * _encoding) : ColumnToOffsetMap(_record, 1, _maxRecordSize, _translateVirtuals)
+CsvColumnToOffsetMap::CsvColumnToOffsetMap(IHqlExpression * _record, unsigned _maxRecordSize, bool _translateVirtuals, IAtom * _encoding)
+: ColumnToOffsetMap(NULL, _record, 0, 1, _maxRecordSize, _translateVirtuals, false)
 {
     encoding = _encoding;
 }
@@ -3297,7 +3481,8 @@ CMemberInfo * CsvColumnToOffsetMap::createColumn(CContainerInfo * container, IHq
 
 //---------------------------------------------------------------------------
 
-XmlColumnToOffsetMap::XmlColumnToOffsetMap(IHqlExpression * _record, unsigned _maxRecordSize, bool _translateVirtuals) : ColumnToOffsetMap(_record, 1, _maxRecordSize, _translateVirtuals)
+XmlColumnToOffsetMap::XmlColumnToOffsetMap(IHqlExpression * _record, unsigned _maxRecordSize, bool _translateVirtuals)
+: ColumnToOffsetMap(NULL, _record, 0, 1, _maxRecordSize, _translateVirtuals, false)
 {
 }
 
@@ -3335,51 +3520,51 @@ BoundRow::BoundRow(const BoundRow & other, IHqlExpression * _newBound) : HqlExpr
     assertex(bound->queryType()->getTypeCode() != type_void);
     resultAlias = false;
     inherited = other.inherited;
+    assertex(!other.accessor); // not currently supported in child queries.
+    accessor.set(other.accessor);
+    accessorStmt = other.accessorStmt;
 }
 
-BoundRow::BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, ColumnToOffsetMap * _columnMap) : HqlExprAssociation(_dataset)
+BoundRow::BoundRow(const BoundRow & other, ColumnToOffsetMap * rawMap, IHqlExpression * _newBound) : HqlExprAssociation(other.represents)
 {
-    assertex(_columnMap);
-    dataset.set(_dataset);
-    bound.set(_bound);
-    columnMap = LINK(_columnMap);
-    conditional = false;
-    side = no_none;
-    kind = AssocRow;
+    dataset.set(other.dataset);
+    bound.set(_newBound ? _newBound : other.bound.get());
+    columnMap = LINK(rawMap);
+    conditional = other.conditional;
+    side = other.side;
+    kind = other.kind;
     assertex(bound->queryType()->getTypeCode() != type_void);
     resultAlias = false;
-    inherited = false;
+    inherited = other.inherited;
+    assertex(!rawMap->usesAccessor());
 }
 
 
-BoundRow::BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, ColumnToOffsetMap * _columnMap, node_operator _side, IHqlExpression * selSeq) : HqlExprAssociation(NULL)
+BoundRow::BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, IHqlExpression * _accessor, ColumnToOffsetMap * _columnMap) : HqlExprAssociation(_dataset)
 {
     assertex(_columnMap);
     dataset.set(_dataset);
     bound.set(_bound);
+    accessor.set(_accessor);
+    assertex(!accessor);
     columnMap = LINK(_columnMap);
     conditional = false;
-    kind = AssocCursor;
-    side = _side;
-    if (side == no_none)
-        represents.set(_dataset->queryNormalizedSelector());
-    else if ((side != no_self) || selSeq)
-        represents.setown(createSelector(side, dataset, selSeq));
-    else
-    {
-        OwnedHqlExpr uid = createUniqueSelectorSequence();
-        represents.setown(createSelector(no_self, dataset, uid));
-    }
+    side = no_none;
+    kind = AssocRow;
     assertex(bound->queryType()->getTypeCode() != type_void);
     resultAlias = false;
     inherited = false;
+    assertex(!accessor == !columnMap->usesAccessor());
 }
 
-BoundRow::BoundRow(BoundRow * _row, IHqlExpression * _dataset, node_operator _side, IHqlExpression * selSeq) : HqlExprAssociation(NULL)
+
+BoundRow::BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, IHqlExpression * _accessor, ColumnToOffsetMap * _columnMap, node_operator _side, IHqlExpression * selSeq) : HqlExprAssociation(NULL)
 {
+    assertex(_columnMap);
     dataset.set(_dataset);
-    bound.set(_row->bound);
-    columnMap = LINK(_row->columnMap);
+    bound.set(_bound);
+    accessor.set(_accessor);
+    columnMap = LINK(_columnMap);
     conditional = false;
     kind = AssocCursor;
     side = _side;
@@ -3395,6 +3580,7 @@ BoundRow::BoundRow(BoundRow * _row, IHqlExpression * _dataset, node_operator _si
     assertex(bound->queryType()->getTypeCode() != type_void);
     resultAlias = false;
     inherited = false;
+//    assertex(!columnMap->usesAccessor());
 }
 
 BoundRow::~BoundRow()
@@ -3437,6 +3623,24 @@ IHqlExpression * BoundRow::queryBuilderEnsureMarker()
     return builderEnsureMarker;
 }
 
+void BoundRow::prepareAccessor(HqlCppTranslator & translator, BuildCtx & ctx)
+{
+    translator.buildRowAccessor(columnMap);
+    StringBuffer className;
+    className.append("access").append(columnMap->queryId());
+    accessor.setown(createVariable(makeConstantModifier(makeClassType(className))));
+
+    OwnedHqlExpr rowPointer = getPointer(bound);
+    accessorStmt = ctx.addDeclare(accessor, rowPointer);
+    accessorStmt->setIncluded(false);
+}
+
+IHqlExpression * BoundRow::ensureAccessor(HqlCppTranslator & translator, BuildCtx & ctx)
+{
+    accessorStmt->setIncluded(true);
+    return accessor;
+}
+
 AColumnInfo * BoundRow::queryRootColumn()   
 { 
     return columnMap->queryRootColumn(); 
@@ -3449,7 +3653,7 @@ unsigned BoundRow::getMaxSize()
 
 //---------------------------------------------------------------------------
 
-NonLocalIndirectRow::NonLocalIndirectRow(const BoundRow & other, IHqlExpression * _newBound, SerializationRow * _serialization) : BoundRow(other, _newBound)
+NonLocalIndirectRow::NonLocalIndirectRow(const BoundRow & other, ColumnToOffsetMap * rawMap, SerializationRow * _serialization) : BoundRow(other, rawMap, nullptr)
 {
     serialization = _serialization;
 }
@@ -3462,7 +3666,7 @@ IHqlExpression * NonLocalIndirectRow::getMappedSelector(BuildCtx & ctx, IReferen
 //---------------------------------------------------------------------------
 
 
-SerializationRow::SerializationRow(HqlCppTranslator & _translator, IHqlExpression * _dataset, IHqlExpression * _bound, DynamicColumnToOffsetMap * _columnMap, ActivityInstance * _activity) : BoundRow(_dataset, _bound, _columnMap, no_none, NULL), translator(_translator)
+SerializationRow::SerializationRow(HqlCppTranslator & _translator, IHqlExpression * _dataset, IHqlExpression * _bound, DynamicColumnToOffsetMap * _columnMap, ActivityInstance * _activity) : BoundRow(_dataset, _bound, NULL, _columnMap, no_none, NULL), translator(_translator)
 {
     serializedMap = _columnMap;
     extractBuilder = NULL;

ecl/hqlcpp/hqltcppc.hpp

@@ -24,9 +24,8 @@ class HQLCPP_API BoundRow : public HqlExprAssociation
 {
 public:
     BoundRow(const BoundRow & other, IHqlExpression * _newBound);                                                       // other row
-    BoundRow(BoundRow * row, IHqlExpression * dataset, node_operator side, IHqlExpression * selSeq);
-    BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, ColumnToOffsetMap * _columnMap);                       // row
-    BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, ColumnToOffsetMap * _columnMap, node_operator side, IHqlExpression * selSeq);  // cursor
+    BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, IHqlExpression * accessor, ColumnToOffsetMap * _columnMap);                       // row
+    BoundRow(IHqlExpression * _dataset, IHqlExpression * _bound, IHqlExpression * accessor, ColumnToOffsetMap * _columnMap, node_operator side, IHqlExpression * selSeq);  // cursor
     ~BoundRow();
 
     virtual AssocKind getKind()             { return (AssocKind)kind; }
@@ -73,10 +72,17 @@ public:
     void setAlias(IReferenceSelector * selector)                { alias.set(selector); }
     IReferenceSelector * queryAlias()                           { return alias; }
     IHqlExpression * queryBuilderEnsureMarker();
+    void prepareAccessor(HqlCppTranslator & translator, BuildCtx & ctx);
+    IHqlExpression * ensureAccessor(HqlCppTranslator & translator, BuildCtx & ctx);
+
+protected:
+    BoundRow(const BoundRow & other, ColumnToOffsetMap * rawMap, IHqlExpression * _newBound); // for colocal child query
 
 protected:
     HqlExprAttr                 dataset;
     HqlExprAttr                 bound;
+    HqlExprAttr                 accessor;
+    IHqlStmt *                  accessorStmt = nullptr;
     HqlExprAttr                 builder;
     HqlExprAttr                 builderEnsureMarker;
     ColumnToOffsetMap *         columnMap;
@@ -143,7 +149,7 @@ protected:
 class NonLocalIndirectRow : public BoundRow
 {
 public:
-    NonLocalIndirectRow(const BoundRow & other, IHqlExpression * _newBound, SerializationRow * _serialization);
+    NonLocalIndirectRow(const BoundRow & other, ColumnToOffsetMap * rawMap, SerializationRow * _serialization);
 
     virtual BoundRow * clone(IHqlExpression * _newBound)    { UNIMPLEMENTED; }
     virtual IHqlExpression * getMappedSelector(BuildCtx & ctx, IReferenceSelector * selector);
@@ -160,6 +166,7 @@ class BoundAliasRow : public BoundRow
 {
 public:
     BoundAliasRow(const BoundRow & other, IHqlExpression * _newBound, IHqlExpression * _expansion) : BoundRow(other, _newBound) { expansion.set(_expansion); }
+    BoundAliasRow(const BoundRow & other, ColumnToOffsetMap * _rawMap, IHqlExpression * _expansion) : BoundRow(other, _rawMap, nullptr) { expansion.set(_expansion); }
     BoundAliasRow(const BoundAliasRow & other, IHqlExpression * _newBound) : BoundRow(other, _newBound) { expansion.set(other.expansion); }
 
     virtual BoundRow * clone(IHqlExpression * _newBound)    { return new BoundAliasRow(*this, _newBound); }

ecl/hqlcpp/hqltcppc.ipp

@@ -43,7 +43,7 @@ public:
     void forceToTemp(node_operator op, IHqlExpression * selector);
     unsigned getFixedSize() const                           { return fixedSize; }
     unsigned getMinimumSize()   const                       { return fixedSize+varMinSize; }
-    IHqlExpression * getSizeExpr(BoundRow * cursor);
+    IHqlExpression * getSizeExpr(BoundRow * cursor) const;
     bool isEmpty() const                                    { return fixedSize == 0 && varSize == NULL; }
     bool isFixedSize() const                                { return varSize == NULL; }
     bool isWorthCommoning() const;
@@ -101,15 +101,20 @@ public:
     virtual unsigned getContainerTrailingFixed();
     virtual bool modifyColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value, node_operator op) { return false; }
     virtual bool checkCompatibleIfBlock(HqlExprCopyArray & conditions);
+    virtual bool bindOffsetsFromClass(SizeStruct & accessorOffset, bool prevVariableSize);
+    virtual void bindSizesFromOffsets(SizeStruct & thisOffset, const SizeStruct & nextOffset);
     
     void addVariableSize(size32_t varMinSize, SizeStruct & size);
     void getXPath(StringBuffer & out);
     StringBuffer & expandSelectPathText(StringBuffer & out, bool isLast) const;
+    IHqlExpression * queryColumn() const { return column; }
+    void getOffsets(SizeStruct & offset, SizeStruct & accessorOffset) const;
 
 public:
     IHqlExpression * getCondition(BuildCtx & ctx);
     IHqlExpression * getConditionSelect(HqlCppTranslator & translator, BuildCtx & ctx, BoundRow * row);
     IHqlExpression * makeConditional(HqlCppTranslator & translator, BuildCtx & ctx, BoundRow * row, IHqlExpression * value);
+
     void setOffset(bool _hasVarOffset);     // to avoid loads of arguments to constructor
 
 protected:
@@ -134,10 +139,10 @@ protected:
     CContainerInfo *    container;
     CMemberInfo *       prior;
     HqlExprAttr         column;
-    HqlExprAttr         varSize;
-    SizeStruct          cachedOffset;
-    SizeStruct          cachedSize;
-//  SizeStruct          cachedMaxSize;
+    SizeStruct          cachedOffset;       // Either fixed or sizeof(x)/offsetof(x) - rebound before building
+    SizeStruct          cachedSize;         // Either fixed or sizeof(x) expressions - rebound before building
+    SizeStruct          cachedAccessorOffset;// A translated expression containing the (fixed, field with the offset)
+    unsigned            seq; // For fields the sequence number, for root container the maximum seq so far
     bool                hasVarOffset;
     bool                isOffsetCached;
 };
@@ -191,9 +196,14 @@ public:
     virtual bool isConditional();
     virtual bool isFixedSize()              { return fixedSize && !isDynamic; }
 
+    virtual bool bindOffsetsFromClass(SizeStruct & accessorOffset, bool prevVariableSize);
+    virtual void bindSizesFromOffsets(SizeStruct & thisOffset, const SizeStruct & nextOffset);
+    virtual bool usesAccessClass() const    { return container->usesAccessClass(); }
+
     void addTrailingFixed(SizeStruct & size, CMemberInfo * cur);
     void subLeadingFixed(SizeStruct & size, CMemberInfo * cur);
     void getContainerXPath(StringBuffer & out);
+    unsigned nextSeq();
     inline unsigned numChildren() const     { return children.ordinality(); }
             
 public:
@@ -207,6 +217,7 @@ protected:
 
 protected:
     CMemberInfoArray    children;
+    SizeStruct          accessorSize;// A translated expression containing the (fixed, field with the offset)
     bool                fixedSize;
     bool                isDynamic;
 };
@@ -225,6 +236,9 @@ public:
 
     virtual IHqlExpression * getRelativeSelf();
     virtual IHqlExpression * queryRootSelf();
+    virtual bool usesAccessClass() const    { return container ? container->usesAccessClass() : useAccessClass; }
+
+    void setUseAccessClass()                { useAccessClass = true; }
 
 protected:
     virtual void registerChild(CMemberInfo * child);
@@ -232,6 +246,7 @@ protected:
 protected:
     ColumnToInfoMap map;
     OwnedHqlExpr cachedSelf;
+    bool useAccessClass;
 };
 
 
@@ -536,9 +551,9 @@ enum MapFormat { MapFormatBinary, MapFormatCsv, MapFormatXml };
 class HQLCPP_API ColumnToOffsetMap : public MappingBase
 {
 public:
-    ColumnToOffsetMap(IHqlExpression * record, unsigned _packing, unsigned _maxRecordSize, bool _translateVirtuals);
+    ColumnToOffsetMap(IHqlExpression * _key, IHqlExpression * record, unsigned _id, unsigned _packing, unsigned _maxRecordSize, bool _translateVirtuals, bool _useAccessClass);
 
-    virtual const void * getKey() const { return &record; }
+    virtual const void * getKey() const { return &key; }
 
     void init(RecordOffsetMap & map);
     unsigned getFixedRecordSize();
@@ -550,9 +565,13 @@ public:
     size32_t getTotalMinimumSize()                  { return root.getTotalMinimumSize(); }
     virtual MapFormat getFormat()                   { return MapFormatBinary; }
     bool queryContainsIfBlock()                     { return containsIfBlock; }
+    IHqlExpression * queryRecord() const            { return record; }
+    unsigned queryId() const                        { return id; }
+    bool usesAccessor() const                       { return root.usesAccessClass(); }
 
     AColumnInfo * queryRootColumn();
     bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper);
+    void buildAccessor(StringBuffer & accessorName, HqlCppTranslator & translator, BuildCtx & declarectx, IHqlExpression * selector);
 
 protected:
     virtual CMemberInfo * addColumn(CContainerInfo * container, IHqlExpression * column, RecordOffsetMap & map);
@@ -562,6 +581,8 @@ protected:
     void ensureMaxSizeCached();
 
 protected:
+    unsigned id;
+    LinkedHqlExpr key;
     IHqlExpression * record;
     CMemberInfo * prior;
     BitfieldPacker packer;

ecl/hqlcpp/hqltcppc2.cpp

@@ -224,7 +224,7 @@ bool CMemberInfo::hasDatasetLimits() const
 
 CChildDatasetColumnInfo::CChildDatasetColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column, RecordOffsetMap & map, unsigned defaultMaxRecordSize) : CColumnInfo(_container, _prior, _column)
 {
-    ColumnToOffsetMap * offsetMap = map.queryMapping(column->queryRecord(), defaultMaxRecordSize);
+    ColumnToOffsetMap * offsetMap = map.queryMapping(column->queryRecord(), defaultMaxRecordSize, false);
     maxChildSize = offsetMap->getMaxSize();
 #ifdef _DEBUG
     assertex(!recordRequiresSerialization(column->queryRecord(), internalAtom));
@@ -397,7 +397,7 @@ CChildLimitedDatasetColumnInfo::CChildLimitedDatasetColumnInfo(CContainerInfo *
         countField.setown(ensureExprType(countField, sizetType));
     if (sizeField)
         sizeField.setown(ensureExprType(sizeField, sizetType));
-    ColumnToOffsetMap * offsetMap = map.queryMapping(column->queryRecord(), defaultMaxRecordSize);
+    ColumnToOffsetMap * offsetMap = map.queryMapping(column->queryRecord(), defaultMaxRecordSize, false);
     maxChildSize = offsetMap->getMaxSize();
     fixedChildSize = offsetMap->isFixedWidth() ? maxChildSize : UNKNOWN_LENGTH;
 }
@@ -621,7 +621,7 @@ AColumnInfo * CChildLimitedDatasetColumnInfo::lookupColumn(IHqlExpression * sear
 
 CChildLinkedDatasetColumnInfo::CChildLinkedDatasetColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column, RecordOffsetMap & map, unsigned defaultMaxRecordSize) : CColumnInfo(_container, _prior, _column)
 {
-    ColumnToOffsetMap * offsetMap = map.queryMapping(column->queryRecord(), defaultMaxRecordSize);
+    ColumnToOffsetMap * offsetMap = map.queryMapping(column->queryRecord(), defaultMaxRecordSize, false);
     maxChildSize = offsetMap->getMaxSize();
 }
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -4411,8 +4411,7 @@ void CompoundSourceTransformer::analyseGatherInfo(IHqlExpression * expr)
                                 IHqlExpression * root = queryRoot(expr);
                                 if (root)
                                 {
-                                    ColumnToOffsetMap * map = translator.queryRecordOffsetMap(root->queryRecord());
-                                    if (map->isFixedWidth())
+                                    if (translator.isFixedRecordSize(root->queryRecord()))
                                         extra->forceCompound = true;
                                 }
                             }

ecl/hqlcpp/hqlwcpp.cpp

@@ -1996,7 +1996,12 @@ void HqlCppWriter::generateStmtDeclare(IHqlStmt * declare)
     if (hasModifier(type, typemod_mutable))
         out.append("mutable ");
 
+    //The following is correct, but causes lots of problems because const isn't currently correctly tracked
+    //if (hasModifier(type, typemod_const))
+    //    out.append("const ");
+
     size32_t typeSize = type->getSize();
+    bool useConstructor = false;
     if (hasWrapperModifier(type))
     {
         ITypeInfo * builderModifier = queryModifier(type, typemod_builder);
@@ -2029,23 +2034,28 @@ void HqlCppWriter::generateStmtDeclare(IHqlStmt * declare)
             out.append("rtlFixedSizeDataAttr<").append(typeSize).append("> ").append(targetName);
         else
             out.append("rtlDataAttr ").append(targetName);
-        if (value)
-        {
-            out.append("(");
-            generateExprCpp(value);
-            out.append(")");
-            value = NULL;
-        }
+        useConstructor = true;
     }
     else
     {
         generateType(type, targetName.str());
+        if (type->getTypeCode() == type_class)
+            useConstructor = true;
     }
 
     if (value)
     {
-        out.append(" = ");
-        generateExprCpp(value);
+        if (useConstructor)
+        {
+            out.append("(");
+            generateExprCpp(value);
+            out.append(")");
+        }
+        else
+        {
+            out.append(" = ");
+            generateExprCpp(value);
+        }
     }
     out.append(";");
     

rtl/eclrtl/CMakeLists.txt

@@ -103,6 +103,7 @@ if (NOT PLUGIN)
     ${CMAKE_CURRENT_SOURCE_DIR}/rtlds_imp.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/rtlfield.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/rtlkey.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/rtlrecord.hpp
   )
     install ( FILES ${iFILES} DESTINATION componentfiles/cl/include COMPONENT Runtime )
   ENDFOREACH ( iFILES )

rtl/eclrtl/eclinclude4.hpp

@@ -71,5 +71,6 @@ typedef unsigned __int64 hash64_t;
 #include "rtlds_imp.hpp"
 #include "eclhelper_base.hpp"
 #include "rtlbcd.hpp"
+#include "rtlrecord.hpp"
 
 #endif

rtl/eclrtl/rtlrecord.cpp

@@ -90,12 +90,66 @@ static unsigned countFields(const RtlFieldInfo * const * fields)
     return cnt;
 }
 
+static unsigned countFields(const RtlFieldInfo * const * fields, bool & containsNested)
+{
+    unsigned cnt = 0;
+    for (;*fields;fields++)
+    {
+        const RtlTypeInfo * type = (*fields)->type;
+        if (type->getType() == type_record)
+        {
+            containsNested = true;
+            const RtlFieldInfo * const * nested = type->queryFields();
+            if (nested)
+                cnt += countFields(nested, containsNested);
+        }
+        else
+            cnt++;
+    }
+    return cnt;
+}
+
+
+static const RtlFieldInfo * * expandNestedRows(const RtlFieldInfo * * target, const RtlFieldInfo * const * fields)
+{
+    for (;*fields;fields++)
+    {
+        const RtlFieldInfo * cur = *fields;
+        const RtlTypeInfo * type = cur->type;
+        if (type->getType() == type_record)
+        {
+            const RtlFieldInfo * const * nested = type->queryFields();
+            if (nested)
+                target = expandNestedRows(target, nested);
+        }
+        else
+            *target++ = cur;
+    }
+    return target;
+}
 
-RtlRecord::RtlRecord(const RtlRecordTypeInfo & record) : fields(record.fields)
+
+RtlRecord::RtlRecord(const RtlRecordTypeInfo & record, bool expandFields) : fields(record.fields), originalFields(record.fields)
 {
     //MORE: Does not cope with ifblocks.
     numVarFields = 0;
-    numFields = countFields(fields);
+    //Optionally expand out nested rows.
+    if (expandFields)
+    {
+        bool containsNested = false;
+        numFields = countFields(fields, containsNested);
+        if (containsNested)
+        {
+            const RtlFieldInfo * * allocated  = new const RtlFieldInfo * [numFields+1];
+            fields = allocated;
+            const RtlFieldInfo * * target = expandNestedRows(allocated, originalFields);
+            assertex(target == fields+numFields);
+            *target = nullptr;
+        }
+    }
+    else
+        numFields = countFields(fields);
+
     for (unsigned i=0; i < numFields; i++)
     {
         if (!queryType(i)->isFixedSize())
@@ -133,6 +187,8 @@ RtlRecord::RtlRecord(const RtlRecordTypeInfo & record) : fields(record.fields)
 
 RtlRecord::~RtlRecord()
 {
+    if (fields != originalFields)
+        delete [] fields;
     delete [] fixedOffsets;
     delete [] whichVariableOffset;
     delete [] variableFieldIds;
@@ -165,12 +221,12 @@ size32_t RtlRecord::getMinRecordSize() const
 
 //---------------------------------------------------------------------------------------------------------------------
 
-RtlRow::RtlRow(const RtlRecord & _info, const void * optRow, size_t * _variableOffsets) : info(_info), variableOffsets(_variableOffsets)
+RtlRow::RtlRow(const RtlRecord & _info, const void * optRow, unsigned numOffsets, size_t * _variableOffsets) : info(_info), variableOffsets(_variableOffsets)
 {
+    assertex(numOffsets == info.getNumVarFields()+1);
     //variableOffset[0] is used for all fixed offset fields to avoid any special casing.
     variableOffsets[0] = 0;
-    if (optRow)
-        setRow(optRow);
+    setRow(optRow);
 }
 
 __int64 RtlRow::getInt(unsigned field) const
@@ -195,7 +251,7 @@ void RtlRow::setRow(const void * _row)
 }
 
 
-RtlDynRow::RtlDynRow(const RtlRecord & _info, const void * optRow) : RtlRow(_info, optRow, new size_t[_info.getNumVarFields()+1])
+RtlDynRow::RtlDynRow(const RtlRecord & _info, const void * optRow) : RtlRow(_info, optRow, _info.getNumVarFields()+1, new size_t[_info.getNumVarFields()+1])
 {
 }
 

rtl/eclrtl/rtlrecord.hpp

@@ -33,7 +33,7 @@ struct ECLRTL_API RtlRecord
 {
 public:
     friend class RtlRow;
-    RtlRecord(const RtlRecordTypeInfo & fields);
+    RtlRecord(const RtlRecordTypeInfo & fields, bool expandFields);
     ~RtlRecord();
 
     void calcRowOffsets(size_t * variableOffsets, const void * _row) const;
@@ -66,12 +66,13 @@ protected:
     unsigned numFields;
     unsigned numVarFields;
     const RtlFieldInfo * const * fields;
+    const RtlFieldInfo * const * originalFields;
 };
 
 struct ECLRTL_API RtlRow
 {
 public:
-    RtlRow(const RtlRecord & _info, const void * optRow, size_t * _variableOffsets);
+    RtlRow(const RtlRecord & _info, const void * optRow, unsigned numOffsets, size_t * _variableOffsets);
 
     __int64 getInt(unsigned field) const;
     void getUtf8(size32_t & resultLen, char * & result, unsigned field) const;
@@ -90,14 +91,14 @@ public:
 
 protected:
     const RtlRecord & info;
-    const void * row = nullptr;
+    const void * row;
     size_t * variableOffsets;       // [0 + 1 entry for each variable size field ]
 };
 
 struct ECLRTL_API RtlDynRow : public RtlRow
 {
 public:
-    RtlDynRow(const RtlRecord & _info, const void * optRow = nullptr);
+    RtlDynRow(const RtlRecord & _info, const void * optRow = NULL);
     ~RtlDynRow();
 };
 
@@ -106,21 +107,23 @@ template <unsigned NUM_VARIABLE_FIELDS>
 struct ECLRTL_API RtlStaticRow : RtlRow
 {
 public:
-    RtlStaticRow(const RtlRecord & _info, const void * optRow = nullptr) : RtlRow(_info, optRow, &offsets) {}
+    RtlStaticRow(const RtlRecord & _info, const void * optRow = NULL) : RtlRow(_info, optRow, NUM_VARIABLE_FIELDS+1, off) {}
 public:
-    size_t offsets[NUM_VARIABLE_FIELDS+1];
+    size_t off[NUM_VARIABLE_FIELDS+1];
 };
 
-class ECLRTL_API RtlRecordSize : CInterfaceOf<IRecordSize>
+class ECLRTL_API RtlRecordSize : public IRecordSize, public RtlCInterface
 {
-    RtlRecordSize(const RtlRecordTypeInfo & fields) : offsetInformation(fields) {}
+public:
+    RtlRecordSize(const RtlRecordTypeInfo & fields) : offsetInformation(fields, true) {}
+    RTLIMPLEMENT_IINTERFACE
 
     virtual size32_t getRecordSize(const void * row)
     {
-        assertex(row);
         //Allocate a temporary offset array on the stack to avoid runtime overhead.
-        size_t * variableOffsets = (size_t *)alloca((offsetInformation.getNumVarFields() + 1) * sizeof(size_t));
-        RtlRow offsetCalculator(offsetInformation, row, variableOffsets);
+        unsigned numOffsets = offsetInformation.getNumVarFields() + 1;
+        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow offsetCalculator(offsetInformation, row, numOffsets, variableOffsets);
         return offsetCalculator.getRecordSize();
     }
 

rtl/include/eclhelper.hpp

@@ -36,6 +36,7 @@ if the supplied pointer was not from the roxiemem heap. Usually an OwnedRoxieStr
 #ifndef CHEAP_UCHAR_DEF
 #include "unicode/utf.h"
 #endif
+#include "rtlconst.hpp"
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
@@ -378,6 +379,7 @@ struct RtlTypeInfo : public RtlITypeInfo
     inline unsigned getBitfieldIntSize() const { return (length & 0xff); }
     inline unsigned getBitfieldNumBits() const { return (length >> 8) & 0xff; }
     inline unsigned getBitfieldShift() const { return (length >> 16) & 0xff; }
+    inline unsigned getType() const { return (fieldType & RFTMkind); }
 
 public:
     unsigned fieldType;

system/include/CMakeLists.txt

@@ -14,3 +14,4 @@
 #    limitations under the License.
 ################################################################################
 Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/platform.h DESTINATION componentfiles/cl/include COMPONENT Runtime )
+Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/rtlconst.hpp DESTINATION componentfiles/cl/include COMPONENT Runtime )

system/include/rtlconst.hpp

@@ -0,0 +1,87 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _RTLCONST_H_
+#define _RTLCONST_H_
+
+// This file contains system wide constants used by runtime and compile time components
+// NB: Do not change the values here they must remain the same
+// Add new types to the end
+
+enum type_vals
+{
+    type_boolean        = 0,
+    type_int            = 1,
+    type_real           = 2,
+    type_decimal        = 3,
+    type_string         = 4,
+    type_alias          = 5, // This is only used when serializing expression graphs
+    type_date           = 6,
+type_unused2            = 7,
+type_unused3            = 8,
+    type_bitfield       = 9,
+type_unused4            = 10,
+    type_char           = 11,
+    type_enumerated     = 12,
+    type_record         = 13,
+    type_varstring      = 14,
+    type_blob           = 15,
+    type_data           = 16,
+    type_pointer        = 17,
+    type_class          = 18,
+    type_array          = 19,
+    type_table          = 20,
+    type_set            = 21,
+    type_row            = 22,
+    type_groupedtable   = 23,
+    type_void           = 24,
+    type_alien          = 25,
+    type_swapint        = 26,
+    type_none           = 27,
+    type_packedint      = 28,
+type_unused5            = 29,
+    type_qstring        = 30,
+    type_unicode        = 31,
+    type_any            = 32,
+    type_varunicode     = 33,
+    type_pattern        = 34,
+    type_rule           = 35,
+    type_token          = 36,
+    type_feature        = 37,
+    type_event          = 38,
+    type_null           = 39,       // not the same as type_void, which should be reserved for actions.
+    type_scope          = 40,
+    type_utf8           = 41,
+    type_transform      = 42,
+    type_ifblock        = 43,       // not a real type -but used for the rtlfield serialization
+    type_function       = 44,
+    type_sortlist       = 45,
+    type_dictionary     = 46,
+
+    type_max,
+
+    type_modifier       = 0xff,     // used by getKind()
+    type_unsigned       = 0x100,  // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
+    type_ebcdic         = 0x200,   // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
+
+//Some pseudo types - never actually created
+    type_stringorunicode= 0xfc, // any string/unicode variant
+    type_numeric        = 0xfd,
+    type_scalar         = 0xfe,
+};
+
+#endif

testing/regress/ecl/key/project2.xml

@@ -0,0 +1,5 @@
+<Dataset name='Result 1'>
+ <Row><id>1</id><hint>Gavin</hint><child><id>3</id><name>no</name><value>12</value></child></Row>
+ <Row><id>2</id><hint>James</hint><child><id>5</id><name>yes</name><value>1</value></child></Row>
+ <Row><id>3</id><hint>Jane</hint><child><id>3</id><name>madness</name><value>12</value></child></Row>
+</Dataset>

testing/regress/ecl/project2.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+childRec := RECORD
+    UNSIGNED id;
+    STRING name;
+    unsigned value;
+END;
+
+r := RECORD
+   unsigned id;
+   STRING hint;
+   childRec child;
+END;
+
+
+ds := DATASET([
+            { 1, 'Gavin', { 3, 'no', 12 } },
+            { 2, 'James', { 5, 'yes', 1 } },
+            { 3, 'Jane', { 3, 'madness', 12 } }], r);
+
+p := PROJECT(NOFOLD(ds), TRANSFORM(r, SELF.id := COUNTER, SELF := LEFT));
+
+OUTPUT(NOFOLD(p));