HPCC-18847 Enable prefetch for alien types and ifblocks

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/thorhelper/thorcommon.ipp

@@ -280,7 +280,7 @@ public:
     {
     }
 
-    virtual void readAhead(IRowDeserializerSource & in)
+    virtual void readAhead(IRowPrefetcherSource & in)
     {
         in.skip(offset);
         original->readAhead(in);
@@ -421,7 +421,7 @@ public:
     {
     }
 
-    virtual void readAhead(IRowDeserializerSource & in)
+    virtual void readAhead(IRowPrefetcherSource & in)
     {
         original->readAhead(in);
         in.skip(offset);

ecl/hql/hqlutil.cpp

@@ -9648,7 +9648,7 @@ void getFieldTypeInfo(FieldTypeInfoStruct &out, ITypeInfo *type)
         }
         else
         {
-            out.fieldType |= (RFTMalien|RFTMinvalidxml|RFTMnoserialize|RFTMnoprefetch);
+            out.fieldType |= (RFTMalien|RFTMinvalidxml|RFTMnoserialize);
             out.fieldType |= RFTMunknownsize;
             //can't work out the size of the field - so keep it as unknown for the moment.
             //until the alien field type is supported
@@ -9799,11 +9799,14 @@ void getFieldTypeInfo(FieldTypeInfoStruct &out, ITypeInfo *type)
         out.locale = str(type->queryLocale());
         out.length = type->getStringLen();
         break;
+    case type_alien:
+        out.className = "RtlAlienTypeInfo";
+        out.fieldType |= (RFTMcontainsunknown|RFTMinvalidxml|RFTMnoserialize);
+        break;
     case type_pointer:
     case type_class:
     case type_array:
     case type_void:
-    case type_alien:
     case type_none:
     case type_any:
     case type_pattern:
@@ -9815,9 +9818,8 @@ void getFieldTypeInfo(FieldTypeInfoStruct &out, ITypeInfo *type)
     case type_scope:
     case type_transform:
     default:
-        out.className = "RtlUnimplementedTypeInfo";
-        out.fieldType |= (RFTMcontainsunknown|RFTMinvalidxml|RFTMnoserialize|RFTMnoprefetch);
-        break;
+        //Type information should not be generated for records containing any of the types above.
+        throwUnexpected();
     }
 }
 
@@ -9836,7 +9838,7 @@ unsigned buildRtlRecordFields(IRtlFieldTypeDeserializer &deserializer, unsigned
         switch (field->getOperator())
         {
         case no_ifblock:
-            typeFlags |= (RFTMunknownsize|RFTMnoprefetch);
+            typeFlags |= (RFTMunknownsize);
             break;
         case no_field:
         {
@@ -9967,6 +9969,12 @@ const RtlTypeInfo *buildRtlType(IRtlFieldTypeDeserializer &deserializer, ITypeIn
     case type_keyedint:
         info.childType = buildRtlType(deserializer, type->queryChildType());
         break;
+    case type_alien:
+    {
+        ITypeInfo * physicalType = queryAlienType(type)->queryPhysicalType();
+        info.childType = buildRtlType(deserializer, physicalType);
+        break;
+    }
     }
     if (info.childType)
         info.fieldType |= info.childType->fieldType & RFTMinherited;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3558,12 +3558,15 @@ bool HqlCppTranslator::buildMetaPrefetcherClass(BuildCtx & ctx, IHqlExpression *
     OwnedHqlExpr dataset = createDataset(no_null, LINK(record));
     bool ok;
     {
-        MemberFunction func(*this, prefetcher.startctx, "virtual void readAhead(IRowDeserializerSource & in) override");
+        MemberFunction func(*this, prefetcher.startctx, "virtual void readAhead(IRowPrefetcherSource & in) override");
+        BoundRow * row = bindTableCursor(func.ctx, dataset, "in.querySelf()", false, no_self, NULL);
         OwnedHqlExpr helper = createVariable("in", makeBoolType());
 
-        ok = queryRecordOffsetMap(record, false)->buildReadAhead(*this, func.ctx, helper);
+        Owned<IReferenceSelector> selector = buildActiveRow(func.ctx, row->querySelector());
+        ok = queryRecordOffsetMap(record, false)->buildReadAhead(*this, func.ctx, selector, helper);
     }
 
+
     if (ok)
     {
         prefetcher.setIncomplete(false);
@@ -3803,7 +3806,7 @@ unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHq
     BuildCtx declarectx(*code, declareAtom);
 
     //First generate a pseudo type entry for an ifblock.
-    unsigned fieldType = type_ifblock|RFTMunknownsize|RFTMnoprefetch;
+    unsigned fieldType = type_ifblock|RFTMunknownsize;
     {
         unsigned length = 0;
         StringBuffer childTypeName;
@@ -3981,6 +3984,10 @@ unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo *
     case type_utf8:
         arguments.append(", \"").append(info.locale).append("\"").toLowerCase();
         break;
+    case type_alien:
+        arguments.append(",&");
+        childType = buildRtlType(arguments, queryAlienType(type)->queryPhysicalType());
+        break;
     }
     info.fieldType |= (childType & RFTMinherited);
 

ecl/hqlcpp/hqlstmt.cpp

@@ -1602,7 +1602,7 @@ private:
 
 //Always convert rtlWriteInt(rtlReadInt()) the inline memcpy is going to be much better.
 //It should probably be converted earlier....
-bool isAwkwardIntSize(IHqlExpression * size)
+static bool isAwkwardIntSize(IHqlExpression * size)
 {
     IValue * value = size->queryValue();
     if (value)
@@ -1619,6 +1619,19 @@ bool isAwkwardIntSize(IHqlExpression * size)
     return false;
 }
 
+static bool isConstOrVar(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_constant:
+    case no_variable:
+        return true;
+    case no_add:
+    case no_mul:
+        return isConstOrVar(expr->queryChild(0)) && isConstOrVar(expr->queryChild(1));
+    }
+    return false;
+}
 
 bool SpecialFunction::canOptimize() const
 {
@@ -1967,6 +1980,9 @@ bool SpecialFunction::queryCombine(const SpecialFunction & next, bool memsetOnly
     {
         if (src != next.src)
             return false;
+        //Only combine skips which are constants, or variables explicitly read from the file - not expressions that may be dependent on querySelf()
+        if (!isConstOrVar(srcLen) || !isConstOrVar(next.srcLen))
+            return false;
         srcLen.setown(peepholeAddExpr(srcLen, next.srcLen));
         return true;
     }

ecl/hqlcpp/hqltcppc.cpp

@@ -1385,24 +1385,14 @@ void CIfBlockInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx,
     CContainerInfo::buildSerialize(translator, condctx, selector, helper, serializeForm);
 }
 
-bool CIfBlockInfo::prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state)
-{
-    gatherSelectExprs(state.requiredValues, condition);
-    return CContainerInfo::prepareReadAhead(translator, state);
-}
-
 bool CIfBlockInfo::buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state)
 {
     try
     {
-        OwnedHqlExpr mappedCondition = quickFullReplaceExpressions(condition, state.requiredValues, state.mappedValues);
-        //Early check to see if all the values have been mapped, rather than relying on exception processing.
-        if (!containsSelector(mappedCondition, queryRootSelf()))
-        {
-            BuildCtx condctx(ctx);
-            translator.buildFilter(condctx, mappedCondition);
-            return CContainerInfo::buildReadAhead(translator, condctx, state);
-        }
+        OwnedHqlExpr mappedCondition = state.selector->queryRootRow()->bindToRow(condition, queryRootSelf());
+        BuildCtx condctx(ctx);
+        translator.buildFilter(condctx, mappedCondition);
+        return CContainerInfo::buildReadAhead(translator, condctx, state);
     }
     catch (IException * e)
     {
@@ -2186,6 +2176,7 @@ IHqlExpression * CAlienColumnInfo::doBuildSizeOfUnbound(HqlCppTranslator & trans
 
     BoundRow * cursor = selector->queryRootRow();
     IHqlExpression * lengthAttr = alien->queryLengthFunction();
+
     if (!lengthAttr->isFunctionDefinition())
     {
         OwnedHqlExpr absoluteLength = replaceSelector(lengthAttr, querySelfReference(), self);
@@ -2229,14 +2220,13 @@ void CAlienColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx
     doBuildDeserialize(translator, ctx, selector, helper, boundSize.expr);
 }
 
-bool CAlienColumnInfo::prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state)
-{
-    return false;  // too complicated to do safetly.  It really needs a rethink...
-}
-
 bool CAlienColumnInfo::buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state)
 {
-    throwUnexpected();
+    OwnedHqlExpr skipSize = doBuildSizeOfUnbound(translator, ctx, state.selector, state.helper);
+    CHqlBoundExpr boundSize;
+    translator.buildExpr(ctx, skipSize, boundSize);
+    callDeserializerSkipInputTranslatedSize(translator, ctx, state.helper, boundSize.expr);
+    return true;
 }
 
 void CAlienColumnInfo::gatherSize(SizeStruct & target)
@@ -3157,11 +3147,11 @@ CMemberInfo * ColumnToOffsetMap::addColumn(CContainerInfo * container, IHqlExpre
 }
 
 
-bool ColumnToOffsetMap::buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper)
+bool ColumnToOffsetMap::buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
 {
     //prepare() allows Ifblock and a count/size on a dataset to tag the fields they depend on which need to be read.
     //The fallback (implemented in base class) is to call deserialize on a temporary row instead.  Ugly, but better than failing.
-    ReadAheadState state(helper);
+    ReadAheadState state(selector, helper);
     if (queryRootColumn()->prepareReadAhead(translator, state))
     {
         state.addDummyMappings();

ecl/hqlcpp/hqltcppc.ipp

@@ -156,7 +156,7 @@ typedef IArrayOf<CMemberInfo> CMemberInfoArray;
 struct ReadAheadState
 {
 public:
-    ReadAheadState(IHqlExpression * _helper) : helper(_helper) {};
+    ReadAheadState(IReferenceSelector * _selector, IHqlExpression * _helper) : helper(_helper), selector(_selector) {};
 
     void addDummyMappings()
     {
@@ -172,6 +172,7 @@ public:
     LinkedHqlExpr helper;
     HqlExprArray requiredValues;
     HqlExprArray mappedValues;
+    IReferenceSelector * selector;
 };
 
 class HQLCPP_API CContainerInfo : public CMemberInfo
@@ -264,7 +265,6 @@ public:
     virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, IAtom * serializeForm);
     virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, IAtom * serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
-    virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
 
     virtual void calcCachedSize(const SizeStruct & offset, SizeStruct & sizeSelf);
@@ -386,7 +386,6 @@ public:
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
     virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, IAtom * serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
-    virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
     virtual bool isFixedSize();
     virtual void gatherSize(SizeStruct & target);
@@ -573,7 +572,7 @@ public:
     bool usesAccessor() const                       { return root.usesAccessClass(); }
 
     AColumnInfo * queryRootColumn();
-    bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper);
+    bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
     void buildAccessor(StringBuffer & accessorName, HqlCppTranslator & translator, BuildCtx & declarectx, IHqlExpression * selector);
 
 protected:

ecl/hqlcpp/hqltcppc2.cpp

@@ -264,8 +264,7 @@ void CColumnInfo::buildDeserializeToBuilder(HqlCppTranslator & translator, Build
     BuildCtx loopctx(ctx);
     buildDeserializeChildLoop(translator, loopctx, selector, helper, serializeForm);
 
-    BoundRow * selfRow = builder->buildDeserializeRow(loopctx, helper, serializeForm);
-    builder->finishRow(loopctx, selfRow);
+    builder->buildDeserializeRow(loopctx, helper, serializeForm);
 }
 
 
@@ -526,13 +525,15 @@ bool CChildLimitedDatasetColumnInfo::buildReadAhead(HqlCppTranslator & translato
             OwnedHqlExpr test = createValue(no_postdec, LINK(bound.expr));
             loopctx.addLoop(test, NULL, false);
 
+            StringBuffer helperCpp;
+            translator.generateExprCpp(helperCpp, state.helper);
             StringBuffer prefetcherInstanceName;
             translator.ensureRowPrefetcher(prefetcherInstanceName, ctx, column->queryRecord());
             
             StringBuffer s;
-            s.append(prefetcherInstanceName).append("->readAhead(");
-            translator.generateExprCpp(s, state.helper).append(");");
-            loopctx.addQuoted(s);
+            loopctx.addQuoted(s.clear().append(helperCpp).append(".noteStartChild();"));
+            loopctx.addQuoted(s.clear().append(prefetcherInstanceName).append("->readAhead(").append(helperCpp).append(");"));
+            loopctx.addQuoted(s.clear().append(helperCpp).append(".noteFinishChild();"));
             return true;
         }
     }

rtl/eclrtl/rtlcommon.cpp

@@ -142,11 +142,15 @@ const byte * CThorContiguousRowBuffer::peek(size32_t maxSize)
 offset_t CThorContiguousRowBuffer::beginNested()
 {
     size32_t len = readSize();
+    //Currently nested datasets are readahead by skipping the number of bytes in the datasets, rather than calling
+    //beginNested(). If this function was ever called from readAhead() then it would need to call noteStartChild()
+    //so that the self pointer is correct for the child rows
     return len+readOffset;
 }
 
 bool CThorContiguousRowBuffer::finishedNested(offset_t & endPos)
 {
+    //See note above, if this was ever called from readAhead() then it would need to call noteFinishChild() and noteStartChild() if incomplete;
     return readOffset >= endPos;
 }
 
@@ -183,3 +187,22 @@ void CThorContiguousRowBuffer::skipVUni()
     ensureAccessible(readOffset+size);
     readOffset += size;
 }
+
+const byte * CThorContiguousRowBuffer::querySelf()
+{
+    if (maxOffset == 0)
+        doPeek(0);
+    if (childStartOffsets.ordinality())
+        return buffer + childStartOffsets.tos();
+    return buffer;
+}
+
+void CThorContiguousRowBuffer::noteStartChild()
+{
+    childStartOffsets.append(readOffset);
+}
+
+void CThorContiguousRowBuffer::noteFinishChild()
+{
+    childStartOffsets.pop();
+}

rtl/eclrtl/rtlcommon.hpp

@@ -9,31 +9,34 @@
 
 //The CThorContiguousRowBuffer is the source for a readAhead call to ensure the entire row
 //is in a contiguous block of memory.  The read() and skip() functions must be implemented
-class ECLRTL_API CThorContiguousRowBuffer : implements IRowDeserializerSource
+class ECLRTL_API CThorContiguousRowBuffer : implements IRowPrefetcherSource
 {
 public:
     CThorContiguousRowBuffer(ISerialStream * _in);
 
     inline void setStream(ISerialStream *_in) { in.set(_in); maxOffset = 0; readOffset = 0; }
 
-    virtual const byte * peek(size32_t maxSize);
-    virtual offset_t beginNested();
-    virtual bool finishedNested(offset_t & len);
-
-    virtual size32_t read(size32_t len, void * ptr);
-    virtual size32_t readSize();
-    virtual size32_t readPackedInt(void * ptr);
-    virtual size32_t readUtf8(ARowBuilder & target, size32_t offset, size32_t fixedSize, size32_t len);
-    virtual size32_t readVStr(ARowBuilder & target, size32_t offset, size32_t fixedSize);
-    virtual size32_t readVUni(ARowBuilder & target, size32_t offset, size32_t fixedSize);
-
-    //These shouldn't really be called since this class is meant to be used for a deserialize.
-    //If we allowed padding/alignment fields in the input then the first function would make sense.
-    virtual void skip(size32_t size);
-    virtual void skipPackedInt();
-    virtual void skipUtf8(size32_t len);
-    virtual void skipVStr();
-    virtual void skipVUni();
+    virtual const byte * peek(size32_t maxSize) override;
+    virtual offset_t beginNested() override;
+    virtual bool finishedNested(offset_t & len) override;
+
+    virtual size32_t read(size32_t len, void * ptr) override;
+    virtual size32_t readSize() override;
+    virtual size32_t readPackedInt(void * ptr) override;
+    virtual size32_t readUtf8(ARowBuilder & target, size32_t offset, size32_t fixedSize, size32_t len) override;
+    virtual size32_t readVStr(ARowBuilder & target, size32_t offset, size32_t fixedSize) override;
+    virtual size32_t readVUni(ARowBuilder & target, size32_t offset, size32_t fixedSize) override;
+
+    //The following functions should only really be called when used by the readAhead() function
+    virtual void skip(size32_t size) override;
+    virtual void skipPackedInt() override;
+    virtual void skipUtf8(size32_t len) override;
+    virtual void skipVStr() override;
+    virtual void skipVUni() override;
+
+    virtual const byte * querySelf() override;
+    virtual void noteStartChild() override;
+    virtual void noteFinishChild() override;
 
     inline bool eos()
     {
@@ -92,6 +95,7 @@ protected:
     const byte * buffer;
     size32_t maxOffset;
     size32_t readOffset;
+    UnsignedArray childStartOffsets;
 };
 
 #endif

rtl/eclrtl/rtldynfield.cpp

@@ -114,6 +114,10 @@ const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_c
     case type_ifblock:
         ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray);
         break;
+    case type_alien:
+        assert(childType);
+        ret = new RtlAlienTypeInfo(fieldType, length, childType);
+        break;
     default:
         throwUnexpected();
     }

rtl/eclrtl/rtlfield.cpp

@@ -205,7 +205,7 @@ size32_t RtlTypeInfoBase::deserialize(ARowBuilder & builder, IRowDeserializerSou
     return offset + thisSize;
 }
 
-void RtlTypeInfoBase::readAhead(IRowDeserializerSource & in) const
+void RtlTypeInfoBase::readAhead(IRowPrefetcherSource & in) const
 {
     size32_t thisSize = size(nullptr, nullptr);
     in.skip(thisSize);
@@ -915,7 +915,7 @@ size32_t RtlPackedIntTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializ
     return offset + size;
 }
 
-void RtlPackedIntTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlPackedIntTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     in.skipPackedInt();
 }
@@ -1142,7 +1142,7 @@ size32_t RtlStringTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerS
     return offset;
 }
 
-void RtlStringTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlStringTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     if (isFixedSize())
     {
@@ -1370,7 +1370,7 @@ size32_t RtlDataTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerSou
     return offset;
 }
 
-void RtlDataTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlDataTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     if (isFixedSize())
     {
@@ -1576,7 +1576,7 @@ size32_t RtlVarStringTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializ
         return offset + in.readVStr(builder, offset, 0);
 }
 
-void RtlVarStringTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlVarStringTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     if (isFixedSize())
     {
@@ -1747,7 +1747,7 @@ size32_t RtlQStringTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializer
     return offset;
 }
 
-void RtlQStringTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlQStringTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     if (isFixedSize())
     {
@@ -2203,7 +2203,7 @@ size32_t RtlUnicodeTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializer
     return offset;
 }
 
-void RtlUnicodeTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlUnicodeTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     if (isFixedSize())
     {
@@ -2414,7 +2414,7 @@ size32_t RtlVarUnicodeTypeInfo::deserialize(ARowBuilder & builder, IRowDeseriali
         return offset + in.readVUni(builder, offset, 0);
 }
 
-void RtlVarUnicodeTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlVarUnicodeTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     if (isFixedSize())
     {
@@ -2542,7 +2542,7 @@ size32_t RtlUtf8TypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerSou
     return offset + sizeof(size32_t) + size;
 }
 
-void RtlUtf8TypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlUtf8TypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     assertex(!isFixedSize());
     size32_t thisLength = in.readSize();
@@ -2679,7 +2679,7 @@ static size32_t deserializeFields(const RtlFieldInfo * const * cur, ARowBuilder
     return offset;
 }
 
-static void readAheadFields(const RtlFieldInfo * const * cur, IRowDeserializerSource & in)
+static void readAheadFields(const RtlFieldInfo * const * cur, IRowPrefetcherSource & in)
 {
     for (;;)
     {
@@ -2791,7 +2791,7 @@ size32_t RtlRecordTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerS
     return deserializeFields(fields, builder, in, offset);
 }
 
-void RtlRecordTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlRecordTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     //Will not generally be called because it will have been expanded
     return readAheadFields(fields, in);
@@ -3011,7 +3011,7 @@ size32_t RtlSetTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerSour
 }
 
 
-void RtlSetTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlSetTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     in.skip(1);
     size32_t thisLength = in.readSize();
@@ -3137,7 +3137,7 @@ size32_t RtlRowTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerSour
 }
 
 
-void RtlRowTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlRowTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     return child->readAhead(in);
 }
@@ -3357,7 +3357,7 @@ size32_t RtlDatasetTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializer
 }
 
 
-void RtlDatasetTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlDatasetTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     size32_t size = in.readSize();
     in.skip(size);
@@ -3573,7 +3573,7 @@ size32_t RtlDictionaryTypeInfo::deserialize(ARowBuilder & builder, IRowDeseriali
     }
 }
 
-void RtlDictionaryTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlDictionaryTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     size32_t size = in.readSize();
     in.skip(size);
@@ -3695,9 +3695,10 @@ size32_t RtlIfBlockTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializer
     return offset;
 }
 
-void RtlIfBlockTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlIfBlockTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
-    UNIMPLEMENTED;
+    if (getCondition(in.querySelf()))
+        readAheadFields(fields, in);
 }
 
 
@@ -3932,7 +3933,7 @@ size32_t RtlUnimplementedTypeInfo::deserialize(ARowBuilder & builder, IRowDeseri
     return offset;
 }
 
-void RtlUnimplementedTypeInfo::readAhead(IRowDeserializerSource & in) const
+void RtlUnimplementedTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
     rtlFailUnexpected();
 }
@@ -3970,6 +3971,91 @@ unsigned RtlUnimplementedTypeInfo::hash(const byte * self, unsigned inhash) cons
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlAlienTypeInfo::getMinSize() const
+{
+    rtlFailUnexpected();
+}
+
+size32_t RtlAlienTypeInfo::size(const byte * self, const byte * selfrow) const
+{
+    if (isFixedSize())
+        return length;
+    rtlFailUnexpected();
+}
+
+size32_t RtlAlienTypeInfo::buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const
+{
+    rtlFailUnexpected();
+}
+
+size32_t RtlAlienTypeInfo::buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const
+{
+    rtlFailUnexpected();
+}
+
+size32_t RtlAlienTypeInfo::buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const
+{
+    rtlFailUnexpected();
+}
+
+size32_t RtlAlienTypeInfo::process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const
+{
+    rtlFailUnexpected();
+}
+
+size32_t RtlAlienTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
+{
+    rtlFailUnexpected();
+}
+
+size32_t RtlAlienTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerSource & in, size32_t offset) const
+{
+    if (isFixedSize())
+        return RtlCompoundTypeInfo::deserialize(builder, in, offset);
+    rtlFailUnexpected();
+}
+
+void RtlAlienTypeInfo::readAhead(IRowPrefetcherSource & in) const
+{
+    if (isFixedSize())
+        in.skip(length);
+    else
+        rtlFailUnexpected();
+}
+
+
+void RtlAlienTypeInfo::getString(size32_t & resultLen, char * & result, const void * ptr) const
+{
+    resultLen = 0;
+    result = nullptr;
+    rtlFailUnexpected();
+}
+
+void RtlAlienTypeInfo::getUtf8(size32_t & resultLen, char * & result, const void * ptr) const
+{
+    resultLen = 0;
+    result = nullptr;
+    rtlFailUnexpected();
+}
+
+__int64 RtlAlienTypeInfo::getInt(const void * ptr) const
+{
+    rtlFailUnexpected();
+    return 0;
+}
+
+int RtlAlienTypeInfo::compare(const byte * left, const byte * right) const
+{
+    rtlFailUnexpected();
+}
+
+unsigned RtlAlienTypeInfo::hash(const byte * self, unsigned inhash) const
+{
+    rtlFailUnexpected();
+}
+
+//-------------------------------------------------------------------------------------------------------------------
+
 RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags, const char *_initializer)
 : RtlFieldInfo(_name, _xpath, _type, _flags, _initializer)
 {

rtl/eclrtl/rtlfield.hpp

@@ -60,7 +60,7 @@ struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
     virtual const RtlTypeInfo * queryChildType() const override;
 
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
 protected:
     size32_t buildUtf8ViaString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     void getUtf8ViaString(size32_t & resultLen, char * & result, const void * ptr) const;
@@ -238,7 +238,7 @@ struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -262,7 +262,7 @@ struct ECLRTL_API RtlStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -286,7 +286,7 @@ struct ECLRTL_API RtlDataTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -310,7 +310,7 @@ struct ECLRTL_API RtlVarStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -332,7 +332,7 @@ struct ECLRTL_API RtlQStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -398,7 +398,7 @@ public:
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -426,7 +426,7 @@ public:
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -453,7 +453,7 @@ public:
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -481,7 +481,7 @@ struct ECLRTL_API RtlRecordTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -518,7 +518,7 @@ struct ECLRTL_API RtlSetTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual int compare(const byte * left, const byte * right) const override;
     virtual unsigned hash(const byte *self, unsigned inhash) const override;
 };
@@ -533,7 +533,7 @@ struct ECLRTL_API RtlRowTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual int compare(const byte * left, const byte * right) const override;
     virtual unsigned hash(const byte *self, unsigned inhash) const override;
 };
@@ -550,7 +550,7 @@ struct ECLRTL_API RtlDatasetTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual int compare(const byte * left, const byte * right) const override;
     virtual unsigned hash(const byte *self, unsigned inhash) const override;
 };
@@ -568,7 +568,7 @@ struct ECLRTL_API RtlDictionaryTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual int compare(const byte * left, const byte * right) const override;
     virtual unsigned hash(const byte *self, unsigned inhash) const override;
 };
@@ -587,7 +587,7 @@ struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -641,7 +641,7 @@ struct ECLRTL_API RtlUnimplementedTypeInfo : public RtlTypeInfoBase
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -650,21 +650,27 @@ struct ECLRTL_API RtlUnimplementedTypeInfo : public RtlTypeInfoBase
     virtual unsigned hash(const byte *self, unsigned inhash) const override;
 };
 
-/*
-
-struct ECLRTL_API RtlAlienTypeInfo : public RtlTypeInfoBase
+struct ECLRTL_API RtlAlienTypeInfo : public RtlCompoundTypeInfo
 {
 public:
-    constexpr inline RtlAlienTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
+    constexpr inline RtlAlienTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo * _child) : RtlCompoundTypeInfo(_fieldType, _length, _child) {}
     virtual void doDelete() const final override { delete this; }
 
-    virtual size32_t size(const byte * self, const byte * selfrow) const override = 0;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
+    virtual size32_t getMinSize() const override;
+    virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
+    virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
+    virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
+    virtual unsigned hash(const byte *self, unsigned inhash) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
-    virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual void readAhead(IRowPrefetcherSource & in) const override;
 };
-*/
 
 
 //-------------------------------------------------------------------------------------------------------------------

rtl/eclrtl/rtlrecord.cpp

@@ -443,7 +443,7 @@ size32_t RtlRecord::deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource
     return offset + lastFixedSize;
 }
 
-void RtlRecord::readAhead(IRowDeserializerSource & in) const
+void RtlRecord::readAhead(IRowPrefetcherSource & in) const
 {
     // Note - this should not and can not be used when ifblocks present - canprefetch flag should take care of that.
     dbgassertex(numIfBlocks==0);
@@ -538,6 +538,21 @@ size32_t RtlRecord::getRecordSize(const void *_row) const
     return size;
 }
 
+size32_t RtlRecord::calculateOffset(const void *_row, unsigned field) const
+{
+    const byte * row = static_cast<const byte *>(_row);
+    size32_t varoffset = 0;
+    unsigned varFields = whichVariableOffset[field];
+    for (unsigned i = 0; i < varFields; i++)
+    {
+        unsigned fieldIndex = variableFieldIds[i];
+        size32_t offset = fixedOffsets[fieldIndex] + varoffset;
+        size32_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
+        varoffset = offset + fieldSize;
+    }
+    return fixedOffsets[field] + varoffset;
+}
+
 //---------------------------------------------------------------------------------------------------------------------
 
 RtlRow::RtlRow(const RtlRecord & _info, const void * optRow, unsigned numOffsets, size_t * _variableOffsets) : info(_info), variableOffsets(_variableOffsets)
@@ -651,7 +666,7 @@ class CDefaultPrefetcher : public CInterfaceOf<ISourceRowPrefetcher>
 public:
     CDefaultPrefetcher(const RtlRecord & _record) : record(_record) {}
 
-    virtual void readAhead(IRowDeserializerSource & in)
+    virtual void readAhead(IRowPrefetcherSource & in)
     {
         record.readAhead(in);
     }
@@ -718,29 +733,6 @@ protected:
     IOutputMetaData * meta;
 };
 
-class ECLRTL_API CSimpleSourceRowPrefetcher : public ISourceRowPrefetcher, public RtlCInterface
-{
-public:
-    CSimpleSourceRowPrefetcher(IOutputMetaData & _meta, ICodeContext * _ctx, unsigned _activityId)
-    {
-        deserializer.setown(_meta.querySerializedDiskMeta()->createDiskDeserializer(_ctx, _activityId));
-        rowAllocator.setown(_ctx->getRowAllocator(&_meta, _activityId));
-    }
-
-    RTLIMPLEMENT_IINTERFACE
-
-    virtual void readAhead(IRowDeserializerSource & in)
-    {
-        RtlDynamicRowBuilder rowBuilder(rowAllocator);
-        size32_t len = deserializer->deserialize(rowBuilder, in);
-        rtlReleaseRow(rowBuilder.finalizeRowClear(len));
-    }
-
-protected:
-    Owned<IOutputRowDeserializer> deserializer;
-    Owned<IEngineRowAllocator> rowAllocator;
-};
-
 //---------------------------------------------------------------------------
 
 
@@ -754,9 +746,7 @@ ISourceRowPrefetcher * COutputMetaData::createDiskPrefetcher(ICodeContext * ctx,
     ISourceRowPrefetcher * fetcher = defaultCreateDiskPrefetcher(ctx, activityId);
     if (fetcher)
         return fetcher;
-    if (queryTypeInfo()->canPrefetch())
-        return new CDefaultPrefetcher(queryRecordAccessor(true));
-    return new CSimpleSourceRowPrefetcher(*this, ctx, activityId);
+    return new CDefaultPrefetcher(queryRecordAccessor(true));
 }
 
 IOutputRowDeserializer *COutputMetaData::createDiskDeserializer(ICodeContext * ctx, unsigned activityId)

rtl/eclrtl/rtlrecord.hpp

@@ -76,9 +76,9 @@ public:
     inline CSourceRowPrefetcher(unsigned _activityId) { activityId = _activityId; ctx = NULL; }
     RTLIMPLEMENT_IINTERFACE
 
-    inline void onCreate(ICodeContext * _ctx) { ctx = _ctx; }
+    virtual void onCreate(ICodeContext * _ctx) { ctx = _ctx; }
 
-    virtual void readAhead(IRowDeserializerSource & in) override = 0;
+    virtual void readAhead(IRowPrefetcherSource & in) override = 0;
 
 protected:
     ICodeContext * ctx;
@@ -113,7 +113,7 @@ class CFixedSourceRowPrefetcher : public CSourceRowPrefetcher
 public:
     inline CFixedSourceRowPrefetcher(unsigned _activityId, unsigned _fixedSize) : CSourceRowPrefetcher(_activityId) { fixedSize = _fixedSize; }
 
-    virtual void readAhead(IRowDeserializerSource & in) { in.skip(fixedSize); }
+    virtual void readAhead(IRowPrefetcherSource & in) { in.skip(fixedSize); }
 
 protected:
     size32_t fixedSize;
@@ -220,10 +220,11 @@ public:
         return getOffset(variableOffsets, numFields);
     }
     size32_t getRecordSize(const void *data) const;
+    size32_t calculateOffset(const void *_row, unsigned field) const;
 
     size32_t getMinRecordSize() const;
     size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in) const;
-    void readAhead(IRowDeserializerSource & in) const;
+    void readAhead(IRowPrefetcherSource & in) const;
 
     inline unsigned getNumFields() const { return numFields; }
     unsigned getNumKeyedFields() const;

rtl/include/eclhelper.hpp

@@ -291,6 +291,7 @@ interface IRowDeserializerSource
     virtual size32_t readVStr(ARowBuilder & target, size32_t offset, size32_t fixedSize) = 0;
     virtual size32_t readVUni(ARowBuilder & target, size32_t offset, size32_t fixedSize) = 0;
 
+    //Generally called from a prefetcher rather than a deserializer, but could be called from a deserializer that also removed fields.
     virtual void skip(size32_t size) = 0;
     virtual void skipPackedInt() = 0;
     virtual void skipUtf8(size32_t len) = 0;
@@ -298,6 +299,13 @@ interface IRowDeserializerSource
     virtual void skipVUni() = 0;
 };
 
+interface IRowPrefetcherSource : extends IRowDeserializerSource
+{
+    virtual const byte * querySelf() = 0; // What is the address of the start of the row being prefetched (or nested dataset)
+    virtual void noteStartChild() = 0;  // start of reading a child row - ensure that querySelf() refers to the child row
+    virtual void noteFinishChild() = 0; // called when finished reading a child row.
+};
+
 interface IOutputRowSerializer : public IInterface
 {
 public:
@@ -313,7 +321,7 @@ public:
 interface ISourceRowPrefetcher : public IInterface
 {
 public:
-    virtual void readAhead(IRowDeserializerSource & in) = 0;
+    virtual void readAhead(IRowPrefetcherSource & in) = 0;
 };
 
 //This version number covers adding new functions into the metadata interface, and the serialized field/type information
@@ -351,13 +359,12 @@ enum RtlFieldTypeMask
     RFTMhasXpath            = 0x00800000,                   // field has xpath
     RFTMhasInitializer      = 0x01000000,                   // field has initialzer
 
-    RFTMnoprefetch          = 0x08000000,                   // readahead function cannot be derived for this field/record
     RFTMcontainsunknown     = 0x10000000,                   // contains a field of unknown type that we can't process properly
     RFTMinvalidxml          = 0x20000000,                   // cannot be called to generate xml
     RFTMhasxmlattr          = 0x40000000,                   // if specified, then xml output includes an attribute (recursive)
     RFTMnoserialize         = 0x80000000,                   // cannot serialize this typeinfo structure (contains aliens or other nasties)
 
-    RFTMinherited           = (RFTMnoprefetch|RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr|RFTMnoserialize)    // These flags are recursively set on any parent records too
+    RFTMinherited           = (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr|RFTMnoserialize)    // These flags are recursively set on any parent records too
 };
 
 //MORE: Can we provide any more useful information about ifblocks  E.g., a pseudo field?  We can add later if actually useful.
@@ -388,7 +395,7 @@ interface RtlITypeInfo
     virtual double getReal(const void * ptr) const = 0;
     virtual size32_t getMinSize() const = 0;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const = 0;
-    virtual void readAhead(IRowDeserializerSource & in) const = 0;
+    virtual void readAhead(IRowPrefetcherSource & in) const = 0;
     virtual int compare(const byte * left, const byte * right) const = 0;
     virtual unsigned hash(const byte * left, unsigned inHash) const = 0;
 protected:
@@ -403,7 +410,6 @@ struct RtlTypeInfo : public RtlITypeInfo
 
 // Some inline helper functions to avoid having to interpret the flags.
     inline bool canSerialize() const { return (fieldType & RFTMnoserialize) == 0; }
-    inline bool canPrefetch() const { return (fieldType & RFTMnoprefetch) == 0; }
     inline bool isEbcdic() const { return (fieldType & RFTMebcdic) != 0; }
     inline bool isFixedSize() const { return (fieldType & RFTMunknownsize) == 0; }
     inline bool isLinkCounted() const { return (fieldType & RFTMlinkcounted) != 0; }

testing/regress/ecl/alien.ecl

@@ -0,0 +1,71 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import lib_stringlib;
+extractXStringLength(data x, unsigned len) := transfer(((data4)(x[1..len])), unsigned4);
+
+//A pretty weird type example - a length prefixed string, where the number of bytes used for the length is configurable...
+xstring(unsigned len) := type
+    export integer physicallength(data x) := extractXStringLength(x, len)+len;
+    export string load(data x) := (string)x[(len+1)..extractXStringLength(x, len)+len];
+    export data store(string x) := transfer(length(x), data4)[1..len]+(data)x;
+end;
+
+reverseString4 := TYPE
+   shared STRING4 REV(STRING4 S) := S[4] + S[3] + S[2] +S[1];
+   EXPORT STRING4 LOAD(STRING4 S) := REV(S);
+   EXPORT STRING4 STORE(STRING4 S) := REV(S);
+END;
+
+ dstring(string del) := TYPE
+    export integer physicallength(string s) := StringLib.StringUnboundedUnsafeFind(s,del)+length(del)-1;
+    export string load(string s) := s[1..StringLib.StringUnboundedUnsafeFind(s,del)-1];
+    export string store(string s) := s+del; // Untested (vlength output generally broken)
+END;
+
+
+pstring := xstring(1);
+ppstring := xstring(2);
+pppstring := xstring(3);
+nameString := string20;
+
+namesRecord :=
+            RECORD
+pstring         surname;
+nameString      forename;
+pppString       addr;
+reverseString4  extra;
+dstring('!')    extra2;
+            END;
+
+namesTable := dataset([
+        {'Hawthorn','Gavin','Slapdash Lane', 'ABCD', ''},
+        {'Hawthorn','Mia','Slapdash Lane', '1234', 'groan'},
+        {'Smithe','Pru','Ashley Road', 'PLEH', 'grown' },
+        {'X','Z','Mars','!EM ', 'help'}], namesRecord);
+
+saved1 := namesTable : independent(many);
+
+saved2 := saved1 : independent(few);
+
+saved3 := saved2 : independent(many);
+
+saved4 := TABLE(saved3, { saved3, unsigned forceTransform := 0; }) : independent(few);
+
+saved5 := TABLE(saved4, { saved4 } - [forceTransform]);
+
+output(saved5);

testing/regress/ecl/key/alien.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><surname>Hawthorn</surname><forename>Gavin               </forename><addr>Slapdash Lane</addr><extra>ABCD</extra><extra2></extra2></Row>
+ <Row><surname>Hawthorn</surname><forename>Mia                 </forename><addr>Slapdash Lane</addr><extra>1234</extra><extra2>groan</extra2></Row>
+ <Row><surname>Smithe</surname><forename>Pru                 </forename><addr>Ashley Road</addr><extra>PLEH</extra><extra2>grown</extra2></Row>
+ <Row><surname>X</surname><forename>Z                   </forename><addr>Mars</addr><extra>!EM </extra><extra2>help</extra2></Row>
+</Dataset>

testing/regress/ecl/key/nestedif.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><id>1</id><numnested>1</numnested><nestedrecord><Row><hasforename>true</hasforename><hassurname>true</hassurname><forename>Gavin</forename><surname>Hawthorn</surname></Row></nestedrecord></Row>
+ <Row><id>2</id><numnested>2</numnested><nestedrecord><Row><hasforename>false</hasforename><hassurname>false</hassurname></Row><Row><hasforename>true</hasforename><hassurname>false</hassurname><forename>X</forename></Row></nestedrecord></Row>
+ <Row><id>3</id><numnested>3</numnested><nestedrecord><Row><hasforename>true</hasforename><hassurname>false</hassurname><forename>Jim</forename></Row><Row><hasforename>false</hasforename><hassurname>true</hassurname><surname>Jones</surname></Row><Row><hasforename>false</hasforename><hassurname>false</hassurname></Row></nestedrecord></Row>
+ <Row><id>99</id><numnested>0</numnested><nestedrecord></nestedrecord></Row>
+</Dataset>

testing/regress/ecl/nestedif.ecl

@@ -0,0 +1,54 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+//Test weird dataset options e.g., a count for the number of entries, with an ifblock inside the nested dataset.  Yuk!
+
+nestedRecord := RECORD
+    boolean hasForename;
+    boolean hasSurname;
+    IFBLOCK (SELF.hasForename)
+        STRING forename;
+    END;
+    IFBLOCK (SELF.hasSurname)
+        STRING surname;
+    END;
+END;
+
+mainRecord := RECORD
+    UNSIGNED id;
+    UNSIGNED numNested;
+    DATASET(nestedRecord, COUNT(SELF.numNested)) nestedRecord;
+END;
+
+mainTable := dataset([
+        {1, 1, [{true, true, 'Gavin', 'Hawthorn'}] },
+        {2, 2, [{false, false}, { true, false, 'X' } ] },
+        {3, 3, [{true, false, 'Jim'}, { false, true, 'Jones' }, { false, false }]},
+        { 99, 0, []}], mainRecord);
+
+saved1 := mainTable : independent(many);
+
+saved2 := saved1 : independent(few);
+
+saved3 := saved2 : independent(many);
+
+saved4 := TABLE(saved3, { saved3, unsigned forceTransform := 0; }) : independent(few);
+
+saved5 := TABLE(saved4, { saved4 } - [forceTransform]);
+
+output(saved5);

thorlcr/thorutil/thmem.cpp

@@ -2511,7 +2511,7 @@ class COutputMetaWithChildRow : public CSimpleInterface, implements IOutputMetaD
         CPrefetcher(ISourceRowPrefetcher *_childPrefetcher, size32_t _extraSz) : childPrefetcher(_childPrefetcher), extraSz(_extraSz)
         {
         }
-        virtual void readAhead(IRowDeserializerSource &in)
+        virtual void readAhead(IRowPrefetcherSource &in)
         {
             in.skip(extraSz);
             byte b;