HPPC-8551 Refactor the serialization functions

- Introduce two serialization functions in the IOutputMetaData
- Rename and rationalise some of the existing functions
- Start work on dictionary serialization functions
- Distinguish between internal and disk serialation formats
- Clean up serialisation code

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/fileview2/fvsource.cpp

@@ -1053,7 +1053,7 @@ bool FVDataSource::setReturnedInfoFromResult()
     bool isKey = false;
     bool isGrouped = false;     // this isn't strictly true...it could be true for an internal result, but no current flag to test
     returnedMeta.setown(new DataSourceMetaData(returnedRecord, 0, true, isGrouped, 0));
-    transformedRecord.setown(getSimplifiedRecord(returnedRecord, isKey));
+    transformedRecord.setown(getFileViewerRecord(returnedRecord, isKey));
 
     if (!transformedRecord)
     {
@@ -1196,7 +1196,7 @@ IFvDataSourceMetaData * createMetaData(IConstWUResult * wuResult)
     if (!record)
         throw MakeStringException(ERR_FILEVIEW_FIRST+4, "Could not process result schema [%s]", s.str());
 
-    OwnedHqlExpr simplifiedRecord = getSimplifiedRecord(record, false);
+    OwnedHqlExpr simplifiedRecord = getFileViewerRecord(record, false);
     bool isGrouped = false;     // more not sure this is strictly true...
     if (!simplifiedRecord)
         return new DataSourceMetaData(record, 0, true, isGrouped, 0);

common/fileview2/fvwugen.cpp

@@ -75,7 +75,7 @@ IHqlExpression * addOutput(IHqlExpression * dataset)
 IHqlExpression * addSimplifyProject(IHqlExpression * dataset)
 {
     IHqlExpression * record = dataset->queryRecord();
-    IHqlExpression * projectRecord = getSimplifiedRecord(record, false);
+    IHqlExpression * projectRecord = getFileViewerRecord(record, false);
     if (!projectRecord)
         return LINK(dataset);
 

common/thorhelper/thorcommon.cpp

@@ -499,23 +499,44 @@ void CThorDemoRowSerializer::endNested(size32_t sizePos)
 
 
 
-IOutputRowSerializer * CachedOutputMetaData::createRowSerializer(ICodeContext * ctx, unsigned activityId) const
+IOutputRowSerializer * CachedOutputMetaData::createDiskSerializer(ICodeContext * ctx, unsigned activityId) const
 {
-    if (metaFlags & (MDFhasserialize|MDFneedserialize))
-        return meta->createRowSerializer(ctx, activityId);
+    if (metaFlags & (MDFhasserialize|MDFneedserializedisk))
+        return meta->createDiskSerializer(ctx, activityId);
     if (isFixedSize())
         return new CSimpleFixedRowSerializer(getFixedSize());
     return new CSimpleVariableRowSerializer(this);
 }
 
 
-IOutputRowDeserializer * CachedOutputMetaData::createRowDeserializer(ICodeContext * ctx, unsigned activityId) const
+IOutputRowDeserializer * CachedOutputMetaData::createDiskDeserializer(ICodeContext * ctx, unsigned activityId) const
 {
-    if (metaFlags & (MDFhasserialize|MDFneedserialize))
-        return meta->createRowDeserializer(ctx, activityId);
+    if (metaFlags & (MDFhasserialize|MDFneedserializedisk))
+        return meta->createDiskDeserializer(ctx, activityId);
     if (isFixedSize())
         return new CSimpleFixedRowDeserializer(getFixedSize());
-    assertex(!"createRowDeserializer variable meta has no serializer");
+    assertex(!"createDiskDeserializer variable meta has no serializer");
+    //return new CSimpleVariableRowDeserializer(this);
+    return NULL;
+}
+
+IOutputRowSerializer * CachedOutputMetaData::createInternalSerializer(ICodeContext * ctx, unsigned activityId) const
+{
+    if (metaFlags & (MDFhasserialize|MDFneedserializeinternal))
+        return meta->createInternalSerializer(ctx, activityId);
+    if (isFixedSize())
+        return new CSimpleFixedRowSerializer(getFixedSize());
+    return new CSimpleVariableRowSerializer(this);
+}
+
+
+IOutputRowDeserializer * CachedOutputMetaData::createInternalDeserializer(ICodeContext * ctx, unsigned activityId) const
+{
+    if (metaFlags & (MDFhasserialize|MDFneedserializeinternal))
+        return meta->createInternalDeserializer(ctx, activityId);
+    if (isFixedSize())
+        return new CSimpleFixedRowDeserializer(getFixedSize());
+    assertex(!"createInternalDeserializer variable meta has no serializer");
     //return new CSimpleVariableRowDeserializer(this);
     return NULL;
 }
@@ -585,7 +606,7 @@ size32_t cloneRow(ARowBuilder & rowBuilder, const void * row, IOutputMetaData *
     size32_t rowSize = meta->getRecordSize(row);        // TBD could be better?
     byte * self = rowBuilder.ensureCapacity(rowSize, NULL);
     memcpy(self, row, rowSize);
-    if (meta->getMetaFlags() & MDFneedserialize)
+    if (meta->getMetaFlags() & MDFneeddestruct)
     {
         ChildRowLinkerWalker walker;
         meta->walkIndirectMembers(self, walker);
@@ -1090,7 +1111,7 @@ IRowInterfaces *createRowInterfaces(IOutputMetaData *meta, unsigned actid, ICode
         {
             if (serializerlock.lock()) {
                 if (!serializer&&meta) 
-                    serializer.setown(meta->createRowSerializer(context,actid));
+                    serializer.setown(meta->createDiskSerializer(context,actid));
                 serializerlock.unlock();
             }
             return serializer;
@@ -1099,7 +1120,7 @@ IRowInterfaces *createRowInterfaces(IOutputMetaData *meta, unsigned actid, ICode
         {
             if (deserializerlock.lock()) {
                 if (!deserializer&&meta) 
-                    deserializer.setown(meta->createRowDeserializer(context,actid));
+                    deserializer.setown(meta->createDiskDeserializer(context,actid));
                 deserializerlock.unlock();
             }
             return deserializer;
@@ -1171,7 +1192,7 @@ public:
             strm.setown(createFileSerialStream(fileio,_ofs,_len,(size32_t)-1, _tallycrc?&crccb:NULL));
         else
             strm.setown(createFileSerialStream(mmfile,_ofs,_len,_tallycrc?&crccb:NULL));
-        prefetcher.setown(rowif->queryRowMetaData()->createRowPrefetcher(rowif->queryCodeContext(), rowif->queryActivityId()));
+        prefetcher.setown(rowif->queryRowMetaData()->createDiskPrefetcher(rowif->queryCodeContext(), rowif->queryActivityId()));
         if (prefetcher)
             prefetchBuffer.setStream(strm);
         source.setStream(strm);
@@ -1541,7 +1562,7 @@ IExtRowWriter *createRowWriter(IFile *iFile, IRowInterfaces *rowIf, unsigned fla
     OwnedIFileIO iFileIO;
     if (TestRwFlag(flags, rw_compress))
     {
-        size32_t fixedSize = rowIf->queryRowMetaData()->querySerializedMeta()->getFixedSize();
+        size32_t fixedSize = rowIf->queryRowMetaData()->querySerializedDiskMeta()->getFixedSize();
         if (fixedSize && TestRwFlag(flags, rw_grouped))
             ++fixedSize; // row writer will include a grouping byte
         iFileIO.setown(createCompressedFileWriter(iFile, fixedSize, TestRwFlag(flags, rw_extend), TestRwFlag(flags, rw_compressblkcrc), compressor, TestRwFlag(flags, rw_fastlz)));

common/thorhelper/thorcommon.ipp

@@ -116,20 +116,22 @@ public:
 //v1 member functions (can be called on any interface)
     inline unsigned getMetaFlags() const                    { return metaFlags; }
     inline bool needsDestruct() const                       { return (metaFlags & MDFneeddestruct) != 0; }
-    inline bool needsSerialize() const                      { return (metaFlags & MDFneedserialize) != 0; }
+    inline bool needsSerializeDisk() const                 { return (metaFlags & MDFneedserializedisk) != 0; }
     inline void destruct(byte * self)
     {
         if (metaFlags & MDFneeddestruct)
             meta->destruct(self);
     }
 
-    IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId) const;
-    IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) const;
+    IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) const;
+    IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) const;
+    IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) const;
+    IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) const;
 
-    inline IOutputMetaData * querySerializedMeta() const
+    inline IOutputMetaData * querySerializedDiskMeta() const
     {
-        if (metaFlags & MDFneedserialize)
-            return meta->querySerializedMeta();
+        if (metaFlags & MDFneedserializedisk)
+            return meta->querySerializedDiskMeta();
         return meta;
     }
 
@@ -338,7 +340,7 @@ public:
     {
         offset = _offset;
         original = _original;
-        IOutputMetaData * originalSerialized = _original->querySerializedMeta();
+        IOutputMetaData * originalSerialized = _original->querySerializedDiskMeta();
         if (originalSerialized != original)
             serializedMeta.setown(new CPrefixedOutputMeta(_offset, originalSerialized));
     }
@@ -368,24 +370,32 @@ public:
 
     virtual unsigned getMetaFlags() { return original->getMetaFlags(); }
     virtual void destruct(byte * self) { original->destruct(self+offset); }
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)
     {
-        return new CPrefixedRowSerializer(offset, original->createRowSerializer(ctx, activityId));
+        return new CPrefixedRowSerializer(offset, original->createDiskSerializer(ctx, activityId));
     }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) 
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) 
     {
-        return new CPrefixedRowDeserializer(offset, original->createRowDeserializer(ctx, activityId));
+        return new CPrefixedRowDeserializer(offset, original->createDiskDeserializer(ctx, activityId));
     }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) 
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) 
     {
-        return new CPrefixedRowPrefetcher(offset, original->createRowPrefetcher(ctx, activityId));
+        return new CPrefixedRowPrefetcher(offset, original->createDiskPrefetcher(ctx, activityId));
     }
-    virtual IOutputMetaData * querySerializedMeta()
+    virtual IOutputMetaData * querySerializedDiskMeta()
     {
         if (serializedMeta)
             return serializedMeta.get();
         return this;
     }
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId)
+    {
+        return new CPrefixedRowSerializer(offset, original->createInternalSerializer(ctx, activityId));
+    }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId)
+    {
+        return new CPrefixedRowDeserializer(offset, original->createInternalDeserializer(ctx, activityId));
+    }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor)
     {
         original->walkIndirectMembers(self+offset, visitor);
@@ -463,7 +473,7 @@ public:
     CSuffixedOutputMeta(size32_t _offset, IOutputMetaData *_original) : original(_original)
     {
         offset = _offset;
-        IOutputMetaData * originalSerialized = _original->querySerializedMeta();
+        IOutputMetaData * originalSerialized = _original->querySerializedDiskMeta();
         if (originalSerialized != original)
             serializedMeta.setown(new CSuffixedOutputMeta(_offset, originalSerialized));
     }
@@ -491,24 +501,32 @@ public:
 
     virtual unsigned getMetaFlags() { return original->getMetaFlags(); }
     virtual void destruct(byte * self) { original->destruct(self); }
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)
     {
-        return new CSuffixedRowSerializer(offset, original->createRowSerializer(ctx, activityId));
+        return new CSuffixedRowSerializer(offset, original->createDiskSerializer(ctx, activityId));
     }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) 
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) 
     {
-        return new CSuffixedRowDeserializer(offset, original->createRowDeserializer(ctx, activityId));
+        return new CSuffixedRowDeserializer(offset, original->createDiskDeserializer(ctx, activityId));
     }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) 
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) 
     {
-        return new CSuffixedRowPrefetcher(offset, original->createRowPrefetcher(ctx, activityId));
+        return new CSuffixedRowPrefetcher(offset, original->createDiskPrefetcher(ctx, activityId));
     }
-    virtual IOutputMetaData * querySerializedMeta()
+    virtual IOutputMetaData * querySerializedDiskMeta()
     {
         if (serializedMeta)
             return serializedMeta.get();
         return this;
     }
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId)
+    {
+        return new CSuffixedRowSerializer(offset, original->createInternalSerializer(ctx, activityId));
+    }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId)
+    {
+        return new CSuffixedRowDeserializer(offset, original->createInternalDeserializer(ctx, activityId));
+    }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor)
     {
         original->walkIndirectMembers(self, visitor);

ecl/eclagent/eclagent.cpp

@@ -1061,7 +1061,7 @@ void EclAgent::getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRow
         MemoryBuffer datasetBuffer;
         MemoryBuffer2IDataVal result(datasetBuffer);
         r->getResultRaw(result, NULL, NULL);
-        rtlDictionary2RowsetX(tcount, tgt, _rowAllocator, deserializer, datasetBuffer.length(), datasetBuffer.toByteArray());
+        rtlDeserializeDictionary(tcount, tgt, _rowAllocator, deserializer, datasetBuffer.length(), datasetBuffer.toByteArray());
     );
 }
 

ecl/eclagent/eclagent.ipp

@@ -651,10 +651,12 @@ public:
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
     virtual void destruct(byte * self) {}
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputMetaData * querySerializedMeta() { return this; }
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
 };
 

ecl/hql/hqlatoms.cpp

@@ -39,9 +39,11 @@ _ATOM assertConstAtom;
 _ATOM atAtom;
 _ATOM atmostAtom;
 _ATOM _attrAligned_Atom;
+_ATOM _attrDiskSerializedForm_Atom;
+_ATOM _attrInternalSerializedForm_Atom;
 _ATOM _attrLocationIndependent_Atom;
+_ATOM _attrMemorySerializedForm_Atom;
 _ATOM _attrRecordCount_Atom;
-_ATOM _attrSerializedForm_Atom;
 _ATOM _attrSize_Atom;
 _ATOM _attrUnadorned_Atom;
 _ATOM aveAtom;
@@ -421,7 +423,9 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKESYSATOM(attrAligned)->setAttrId(EAaligned);
     MAKESYSATOM(attrLocationIndependent)->setAttrId(EAlocationIndependent);
     MAKESYSATOM(attrRecordCount)->setAttrId(EArecordCount);
-    MAKESYSATOM(attrSerializedForm)->setAttrId(EAserializedForm);
+    MAKESYSATOM(attrDiskSerializedForm)->setAttrId(EAdiskserializedForm);
+    MAKESYSATOM(attrInternalSerializedForm)->setAttrId(EAinternalserializedForm);
+    MAKESYSATOM(attrMemorySerializedForm)->setAttrId(EAmemoryserializedForm);
     MAKESYSATOM(attrSize)->setAttrId(EAsize);
     MAKESYSATOM(attrUnadorned)->setAttrId(EAunadorned);
     MAKEATOM(ave);

ecl/hql/hqlatoms.hpp

@@ -42,9 +42,11 @@ extern HQL_API _ATOM assertConstAtom;
 extern HQL_API _ATOM atAtom;
 extern HQL_API _ATOM atmostAtom;
 extern HQL_API _ATOM _attrAligned_Atom;
+extern HQL_API _ATOM _attrDiskSerializedForm_Atom;
+extern HQL_API _ATOM _attrInternalSerializedForm_Atom;
 extern HQL_API _ATOM _attrLocationIndependent_Atom;
+extern HQL_API _ATOM _attrMemorySerializedForm_Atom;
 extern HQL_API _ATOM _attrRecordCount_Atom;
-extern HQL_API _ATOM _attrSerializedForm_Atom;
 extern HQL_API _ATOM _attrSize_Atom;
 extern HQL_API _ATOM _attrUnadorned_Atom;
 extern HQL_API _ATOM aveAtom;
@@ -386,7 +388,9 @@ enum
 {
     EAnone,
     EArecordCount,
-    EAserializedForm,
+    EAdiskserializedForm,
+    EAinternalserializedForm,
+    EAmemoryserializedForm,
     EAsize,
     EAaligned,
     EAunadorned,

ecl/hql/hqlattr.cpp

@@ -763,11 +763,21 @@ static unsigned guessSize(unsigned minLen, unsigned maxLen)
 }
 
 
-static IHqlExpression * querySerializedForm(IHqlExpression * expr)
+static IHqlExpression * querySerializedForm(IHqlExpression * expr, _ATOM variation)
 {
     if (expr)
     {
-        IHqlExpression * attr = expr->queryAttribute(_attrSerializedForm_Atom);
+        _ATOM attrName;
+        if (variation == diskAtom)
+            attrName = _attrDiskSerializedForm_Atom;
+        else if (variation == internalAtom)
+            attrName = _attrInternalSerializedForm_Atom;
+        else if (variation == memoryAtom)
+            attrName = _attrMemorySerializedForm_Atom;
+        else
+            throwUnexpected();
+
+        IHqlExpression * attr = expr->queryAttribute(attrName);
         if (attr)
             return attr;
     }
@@ -779,7 +789,7 @@ static HqlTransformerInfo serializedRecordCreatorInfo("SerializedRecordCreator")
 class SerializedRecordCreator : public QuickHqlTransformer
 {
 public:
-    SerializedRecordCreator() : QuickHqlTransformer(serializedRecordCreatorInfo, NULL) {}
+    SerializedRecordCreator(_ATOM _variety) : QuickHqlTransformer(serializedRecordCreatorInfo, NULL), variety(_variety) {}
 
     virtual IHqlExpression * createTransformedBody(IHqlExpression * expr)
     {
@@ -797,11 +807,20 @@ public:
         }
         return QuickHqlTransformer::createTransformedBody(expr);
     }
+
+    virtual ITypeInfo * transformType(ITypeInfo * type)
+    {
+        Owned<ITypeInfo> transformed = QuickHqlTransformer::transformType(type);
+        return getSerializedForm(transformed, variety);
+    }
+
+protected:
+    _ATOM variety;
 };
 
-static IHqlExpression * evaluateSerializedRecord(IHqlExpression * expr)
+static IHqlExpression * evaluateSerializedRecord(IHqlExpression * expr, _ATOM variation)
 {
-    SerializedRecordCreator transformer;
+    SerializedRecordCreator transformer(variation);
     return transformer.transform(expr);
 }
 
@@ -825,17 +844,17 @@ public:
 
 //-- Attribute: serialized form -------------------------------------------------------------------------------
 
-static IHqlExpression * evaluateAttrSerializedForm(IHqlExpression * expr)
+static IHqlExpression * evaluateAttrSerializedForm(IHqlExpression * expr, _ATOM attr, _ATOM variation)
 {
     if (expr->getOperator() == no_record || expr->getOperator() == no_field)
     {
-        OwnedHqlExpr serialized = evaluateSerializedRecord(expr);
+        OwnedHqlExpr serialized = evaluateSerializedRecord(expr, variation);
         if (serialized != expr)
         {
             //Tag serialized form so don't re-evaluated
-            meta.addAttribute(serialized, _attrSerializedForm_Atom, serialized);
+            meta.addAttribute(serialized, attr, serialized);
         }
-        return meta.addAttribute(expr, _attrSerializedForm_Atom, serialized);
+        return meta.addAttribute(expr, attr, serialized);
     }
     return NULL;
 }
@@ -1221,12 +1240,12 @@ static IHqlExpression * evaluateAttrSize(IHqlExpression * expr)
 }
 
 
-IHqlExpression * getSerializedForm(IHqlExpression * expr)
+IHqlExpression * getSerializedForm(IHqlExpression * expr, _ATOM variation)
 {
-    return LINK(querySerializedForm(expr));
+    return LINK(querySerializedForm(expr, variation));
 }
 
-ITypeInfo * getSerializedForm(ITypeInfo * type)
+ITypeInfo * getSerializedForm(ITypeInfo * type, _ATOM variation)
 {
     if (!type)
         return NULL;
@@ -1235,22 +1254,34 @@ ITypeInfo * getSerializedForm(ITypeInfo * type)
     case type_record:
         {
             IHqlExpression * record = queryRecord(queryUnqualifiedType(type));
-            OwnedHqlExpr serializedRecord = getSerializedForm(record);
+            IHqlExpression * serializedRecord = querySerializedForm(record, variation);
             if (record == serializedRecord)
                 return LINK(type);
-            return cloneModifiers(type, serializedRecord->queryType());
+            return cloneModifiers(type, serializedRecord->getType());
         }
     case type_row:
     case type_transform:
     case type_table:
     case type_groupedtable:
         {
+            //MORE: If (variant == internalAtom) consider using a format that prefixes the dataset with a count instead of a size
             OwnedITypeInfo noOutOfLineType = removeModifier(type, typemod_outofline);
             OwnedITypeInfo noLinkCountType = removeProperty(noOutOfLineType, _linkCounted_Atom);
             ITypeInfo * childType = noLinkCountType->queryChildType();
-            OwnedITypeInfo newChild = getSerializedForm(childType);
+            OwnedITypeInfo newChild = getSerializedForm(childType, variation);
             return replaceChildType(noLinkCountType, newChild);
         }
+    case type_dictionary:
+        {
+            OwnedITypeInfo noOutOfLineType = removeModifier(type, typemod_outofline);
+            OwnedITypeInfo noLinkCountType = removeProperty(noOutOfLineType, _linkCounted_Atom);
+            ITypeInfo * childType = noLinkCountType->queryChildType();
+            OwnedITypeInfo newChild = getSerializedForm(childType, variation);
+            if (variation == internalAtom)
+                return replaceChildType(noLinkCountType, newChild);
+            OwnedITypeInfo datasetType = makeTableType(LINK(newChild), NULL, NULL, NULL);
+            return cloneModifiers(noLinkCountType, datasetType);
+        }
     }
     return LINK(type);
 }
@@ -1275,6 +1306,7 @@ HqlCachedAttributeTransformer::HqlCachedAttributeTransformer(HqlTransformerInfo
 {
 }
 
+
 IHqlExpression * HqlCachedAttributeTransformer::transform(IHqlExpression * expr)
 {
     IHqlExpression * match = meta.queryExistingAttribute(expr, attrName);
@@ -3202,10 +3234,14 @@ bool maxRecordSizeCanBeDerived(IHqlExpression * record)
 //---------------------------------------------------------------------------------
 
 
-bool recordRequiresSerialization(IHqlExpression * expr)
+bool recordRequiresSerialization(IHqlExpression * expr, _ATOM serializeForm)
 {
-    if (recordRequiresDestructor(expr))
+    if (!expr)
+        return false;
+
+    if (querySerializedForm(expr, serializeForm) != expr)
         return true;
+
     return false;
 }
 
@@ -3214,11 +3250,24 @@ bool recordRequiresDestructor(IHqlExpression * expr)
     if (!expr)
         return false;
 
-    //true if the serialized form is different
-    IHqlExpression * serialized = expr->queryAttribute(_attrSerializedForm_Atom);
-    return serialized && (serialized != expr->queryBody());
+    //true if the internal serialized form is different
+    if (querySerializedForm(expr, internalAtom) != expr)
+        return true;
+
+    return false;
+}
+
+bool recordRequiresLinkCount(IHqlExpression * expr)
+{
+    //MORE: This should strictly speaking check if any of the child fields are link counted.
+    //This function is a sufficient proxy at the moment
+    return recordRequiresDestructor(expr);
 }
 
+bool recordSerializationDiffers(IHqlExpression * expr, _ATOM serializeForm1, _ATOM serializeForm2)
+{
+    return querySerializedForm(expr, serializeForm1) != querySerializedForm(expr, serializeForm2);
+}
 
 //---------------------------------------------------------------------------------
 
@@ -3627,8 +3676,12 @@ IHqlExpression * CHqlExpression::queryAttribute(_ATOM propName)
     {
     case EArecordCount:
         return evaluateAttrRecordCount(this);
-    case EAserializedForm:
-        return evaluateAttrSerializedForm(this);
+    case EAdiskserializedForm:
+        return evaluateAttrSerializedForm(this, _attrDiskSerializedForm_Atom, diskAtom);
+    case EAinternalserializedForm:
+        return evaluateAttrSerializedForm(this, _attrInternalSerializedForm_Atom, internalAtom);
+    case EAmemoryserializedForm:
+        return evaluateAttrSerializedForm(this, _attrMemorySerializedForm_Atom, memoryAtom);
     case EAsize:
         return evaluateAttrSize(this);
     case EAaligned:

ecl/hql/hqlattr.hpp

@@ -39,10 +39,13 @@ extern HQL_API bool increasesRowSize(IHqlExpression * expr);
 extern HQL_API bool isVariableSizeRecord(IHqlExpression * record);
 inline bool isFixedSizeRecord(IHqlExpression * record) { return !isVariableSizeRecord(record); }
 
+extern HQL_API bool recordRequiresLinkCount(IHqlExpression * expr);
 extern HQL_API bool recordRequiresDestructor(IHqlExpression * expr);
-extern HQL_API bool recordRequiresSerialization(IHqlExpression * expr);
-extern HQL_API IHqlExpression * getSerializedForm(IHqlExpression * expr);
-extern HQL_API ITypeInfo * getSerializedForm(ITypeInfo * type);
+//extern HQL_API bool recordRequiresSerialization(IHqlExpression * expr);
+extern HQL_API bool recordRequiresSerialization(IHqlExpression * expr, _ATOM serializeForm);
+extern HQL_API bool recordSerializationDiffers(IHqlExpression * expr, _ATOM serializeForm1, _ATOM serializeForm2);
+extern HQL_API IHqlExpression * getSerializedForm(IHqlExpression * expr, _ATOM variation);
+extern HQL_API ITypeInfo * getSerializedForm(ITypeInfo * type, _ATOM variation);
 extern HQL_API IHqlExpression * getPackedRecord(IHqlExpression * expr);
 extern HQL_API IHqlExpression * getUnadornedExpr(IHqlExpression * expr);
 

ecl/hql/hqlerrors.hpp

@@ -73,6 +73,8 @@
 #define WRN_EXPORT_IGNORED          1048
 #define WRN_RECORDMANYFIELDS        1049
 #define WRN_RESERVED_FUTURE         1050 /* Identifier likely to be reserved in future versions */
+#define WRN_SILLY_EXISTS            1051
+
 //#define ECL_WARN_END          1100
 
 ///////////////////////////////////////////////////////////////////////////////

ecl/hql/hqlexpr.cpp

@@ -1449,7 +1449,7 @@ const char *getOpString(node_operator op)
     case no_childquery: return "no_childquery";
     case no_createdictionary: return "DICTIONARY";
     case no_chooseds: return "CHOOSE";
-    case no_datasetfromdictionary: return "DICTIONARY";
+    case no_datasetfromdictionary: return "DATASET";
 
     case no_unused6:
     case no_unused13: case no_unused14: case no_unused15:
@@ -2157,8 +2157,6 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_graphloop:
     case no_extractresult:
     case no_filtergroup:
-    case no_deserialize:
-    case no_serialize:
     case no_forcegraph:
     case no_executewhen:
     case no_normalizegroup:
@@ -2166,6 +2164,11 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_dataset_alias:
     case no_ensureresult:
         return 1;
+    case no_deserialize:
+    case no_serialize:
+        if (dataset->queryChild(0)->isDataset())
+            return 1;
+        return 0;
     case no_childdataset:
     case no_left:
     case no_right:
@@ -3407,6 +3410,7 @@ void CHqlExpression::updateFlagsAfterOperands()
         infoFlags = (infoFlags & ~HEFassigninheritFlags) | (queryChild(1)->getInfoFlags() & HEFassigninheritFlags);
         infoFlags2 = (infoFlags2 & ~HEF2assigninheritFlags) | (queryChild(1)->getInfoFlags2() & HEF2assigninheritFlags);
         infoFlags &= ~HEFcontainsDataset;
+        assertex(queryChild(0)->isDictionary() == queryChild(1)->isDictionary());
         break;
     case no_delayedselect:
     case no_libraryselect:
@@ -5484,7 +5488,7 @@ void CHqlField::onCreateField()
     case type_groupedtable:
         typeExpr = queryRecord();
 #ifdef _DEBUG
-        assertex(!recordRequiresSerialization(typeExpr) || hasProperty(_linkCounted_Atom));
+        assertex(!recordRequiresLinkCount(typeExpr) || hasProperty(_linkCounted_Atom));
 #endif
         break;
     }
@@ -5748,6 +5752,8 @@ CHqlDictionary *CHqlDictionary::makeDictionary(node_operator _op, ITypeInfo *typ
 CHqlDictionary::CHqlDictionary(node_operator _op, ITypeInfo *_type, HqlExprArray &_ownedOperands)
 : CHqlExpressionWithType(_op, _type, _ownedOperands)
 {
+    if (op == no_select)
+        normalized.setown(calcNormalizedSelector());
 }
 
 CHqlDictionary::~CHqlDictionary()
@@ -5759,6 +5765,17 @@ IHqlExpression *CHqlDictionary::clone(HqlExprArray &newkids)
     return createDictionary(op, newkids);
 }
 
+IHqlExpression * CHqlDictionary::queryNormalizedSelector(bool skipIndex)
+{
+    if (!normalized)
+        return this;
+    if (!skipIndex)
+        return normalized;
+    return normalized->queryNormalizedSelector(skipIndex);
+}
+
+
+
 //===========================================================================
 
 CHqlDataset *CHqlDataset::makeDataset(node_operator _op, ITypeInfo *type, HqlExprArray &_ownedOperands)
@@ -9997,7 +10014,7 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
     {
         IHqlExpression * record = &parms.item(0);
         IHqlExpression * metadata = queryProperty(_metadata_Atom, parms);
-        bool linkCounted = (queryProperty(_linkCounted_Atom, parms) || recordRequiresSerialization(record));
+        bool linkCounted = (queryProperty(_linkCounted_Atom, parms) || recordRequiresLinkCount(record));
         if (!metadata)
         {
             ITypeInfo * recordType = createRecordType(record);
@@ -10012,6 +10029,33 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
             type.setown(setLinkCountedAttr(type, true));
         break;
     }
+    case no_serialize:
+        {
+            assertex(parms.ordinality() >= 2);
+            IHqlExpression & form = parms.item(1);
+            assertex(form.isAttribute());
+            assertex(form.queryName() != diskAtom);  //It should be a dataset instead...
+            type.setown(getSerializedForm(parms.item(0).queryType(), form.queryName()));
+            break;
+        }
+    case no_deserialize:
+        {
+            assertex(parms.ordinality() >= 3);
+            IHqlExpression & record = parms.item(1);
+            IHqlExpression & form = parms.item(2);
+            assertex(form.isAttribute());
+            ITypeInfo * recordType = record.queryType();
+            OwnedITypeInfo rowType = makeRowType(LINK(recordType));
+            assertex(record.getOperator() == no_record);
+            type.setown(makeDictionaryType(LINK(rowType)));
+            type.setown(setLinkCountedAttr(type, true));
+
+    #ifdef _DEBUG
+            OwnedITypeInfo serializedType = getSerializedForm(type, form.queryName());
+            assertex(recordTypesMatch(serializedType, parms.item(0).queryType()));
+    #endif
+            break;
+        }
     case no_nofold:
     case no_nohoist:
     case no_thor:
@@ -10020,6 +10064,7 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
     case no_translated:
     case no_catch:
     case no_colon:
+    case no_globalscope:
         type.set(parms.item(0).queryType());
         break;
     default:
@@ -10954,7 +10999,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             bool matchesAnyMeta =  ((op == no_null) || (op == no_fail));
             IHqlExpression * record = &parms.item(0);
             IHqlExpression * metadata = queryProperty(_metadata_Atom, parms);
-            bool linkCounted = (queryProperty(_linkCounted_Atom, parms) || recordRequiresSerialization(record));
+            bool linkCounted = (queryProperty(_linkCounted_Atom, parms) || recordRequiresLinkCount(record));
             if (!metadata)
             {
                 IHqlExpression * distributed = queryProperty(distributedAtom, parms);
@@ -11747,11 +11792,19 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             break;
         }
     case no_serialize:
-        type.setown(getSerializedForm(datasetType));
-        break;
+        {
+            assertex(parms.ordinality() >= 2);
+            IHqlExpression & form = parms.item(1);
+            assertex(form.isAttribute());
+            type.setown(getSerializedForm(datasetType, form.queryName()));
+            break;
+        }
     case no_deserialize:
         {
+            assertex(parms.ordinality() >= 3);
             IHqlExpression & record = parms.item(1);
+            IHqlExpression & form = parms.item(2);
+            assertex(form.isAttribute());
             ITypeInfo * recordType = record.queryType();
             OwnedITypeInfo rowType = makeRowType(LINK(recordType));
             assertex(record.getOperator() == no_record);
@@ -11768,7 +11821,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             type.setown(setLinkCountedAttr(type, true));
 
 #ifdef _DEBUG
-            OwnedITypeInfo serializedType = getSerializedForm(type);
+            OwnedITypeInfo serializedType = getSerializedForm(type, form.queryName());
             assertex(recordTypesMatch(serializedType, datasetType));
 #endif
             break;
@@ -11932,18 +11985,26 @@ extern IHqlExpression *createRow(node_operator op, HqlExprArray & args)
             type = args.item(2).getType();
         break;
     case no_serialize:
-        type = getSerializedForm(args.item(0).queryType());
-        break;
+        {
+            assertex(args.ordinality() >= 2);
+            IHqlExpression & form = args.item(1);
+            assertex(form.isAttribute());
+            type = getSerializedForm(args.item(0).queryType(), form.queryName());
+            break;
+        }
     case no_deserialize:
         {
+            assertex(args.ordinality() >= 3);
             ITypeInfo * childType = args.item(0).queryType();
             IHqlExpression & record = args.item(1);
+            IHqlExpression & form = args.item(2);
+            assertex(form.isAttribute());
             assertex(record.getOperator() == no_record);
             ITypeInfo * recordType = record.queryType();
             type = makeAttributeModifier(makeRowType(LINK(recordType)), getLinkCountedAttr());
 
 #ifdef _DEBUG
-            OwnedITypeInfo serializedType = getSerializedForm(type);
+            OwnedITypeInfo serializedType = getSerializedForm(type, form.queryName());
             assertex(recordTypesMatch(serializedType, childType));
 #endif
             break;
@@ -11952,7 +12013,7 @@ extern IHqlExpression *createRow(node_operator op, HqlExprArray & args)
         {
             IHqlExpression * record = &args.item(0);
             type = makeRowType(record->getType());
-            if (recordRequiresSerialization(record))
+            if (recordRequiresLinkCount(record))
                 type = makeAttributeModifier(type, getLinkCountedAttr());
             break;
         }
@@ -12406,32 +12467,33 @@ IHqlExpression * ensureActiveRow(IHqlExpression * expr)
     return createRow(no_activerow, LINK(expr->queryNormalizedSelector()));
 }
 
-IHqlExpression * ensureSerialized(IHqlExpression * expr)
+IHqlExpression * ensureSerialized(IHqlExpression * expr, _ATOM serialForm)
 {
-    //MORE: Extend once strings and other items can be stored out of line
-    IHqlExpression * record = expr->queryRecord();
-    if (!record || !recordRequiresSerialization(record))
+    ITypeInfo * type = expr->queryType();
+    Owned<ITypeInfo> serialType = getSerializedForm(type, serialForm);
+    if (type == serialType)
         return LINK(expr);
 
-    return createWrapper(no_serialize, LINK(expr));
+    HqlExprArray args;
+    args.append(*LINK(expr));
+    args.append(*createAttribute(serialForm));
+    return createWrapper(no_serialize, serialType, args);
 }
 
-IHqlExpression * ensureDeserialized(IHqlExpression * expr, ITypeInfo * type)
+IHqlExpression * ensureDeserialized(IHqlExpression * expr, ITypeInfo * type, _ATOM serialForm)
 {
-    //MORE: Extend once strings and other items can be stored out of line
-    IHqlExpression * record = queryRecord(type);
-    if (!record || !recordRequiresSerialization(record))
+    Owned<ITypeInfo> serialType = getSerializedForm(type, serialForm);
+    if (type == serialType)
         return LINK(expr);
 
-    IHqlExpression * exprRecord = expr->queryRecord();
-    OwnedHqlExpr serializedRecord = getSerializedForm(record);
-    assertex(recordTypesMatch(exprRecord, serializedRecord));
+    assertRecordTypesMatch(expr->queryType(), serialType);
 
-    //MORE: I may prefer to create a project instead of a serialize...
     HqlExprArray args;
     args.append(*LINK(expr));
-    args.append(*LINK(record));
-    return createWrapper(no_deserialize, args);
+    args.append(*LINK(queryOriginalRecord(type)));
+    args.append(*createAttribute(serialForm));
+    //MORE: I may prefer to create a project instead of a serialize...
+    return createWrapper(no_deserialize, type, args);
 }
 
 bool isDummySerializeDeserialize(IHqlExpression * expr)
@@ -12448,6 +12510,10 @@ bool isDummySerializeDeserialize(IHqlExpression * expr)
     if (op == childOp)
         return false;
 
+    //MORE:? Need to check the serialization form?
+    if (expr->queryType() != child->queryChild(0)->queryType())
+        return false;
+
     return true;
 }
 
@@ -14088,7 +14154,7 @@ IHqlExpression * querySkipDatasetMeta(IHqlExpression * dataset)
 
 //==============================================================================================================
 
-static ITypeInfo * doGetSimplifiedType(ITypeInfo * type, bool isConditional, bool isSerialized)
+static ITypeInfo * doGetSimplifiedType(ITypeInfo * type, bool isConditional, bool isSerialized, _ATOM serialForm)
 {
     ITypeInfo * promoted = type->queryPromotedType();
     switch (type->getTypeCode())
@@ -14116,29 +14182,29 @@ static ITypeInfo * doGetSimplifiedType(ITypeInfo * type, bool isConditional, boo
     case type_bitfield:
         return LINK(promoted);
     case type_alien:
-        return getSimplifiedType(promoted, isConditional, isSerialized);
+        return getSimplifiedType(promoted, isConditional, isSerialized, serialForm);
     case type_row:
     case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_transform:
         if (isSerialized)
-            return getSerializedForm(type);
+            return getSerializedForm(type, serialForm);
         //should possibly remove some weird options
         return LINK(type);
     case type_set:
         {
             ITypeInfo * childType = type->queryChildType();
-            Owned<ITypeInfo> simpleChildType = getSimplifiedType(childType, false, isSerialized);
+            Owned<ITypeInfo> simpleChildType = getSimplifiedType(childType, false, isSerialized, serialForm);
             return makeSetType(simpleChildType.getClear());
         }
     }
     UNIMPLEMENTED;  //i.e. doesn't make any sense....
 }
 
-ITypeInfo * getSimplifiedType(ITypeInfo * type, bool isConditional, bool isSerialized)
+ITypeInfo * getSimplifiedType(ITypeInfo * type, bool isConditional, bool isSerialized, _ATOM serialForm)
 {
-    Owned<ITypeInfo> newType = doGetSimplifiedType(type, isConditional, isSerialized);
+    Owned<ITypeInfo> newType = doGetSimplifiedType(type, isConditional, isSerialized, serialForm);
     if (isSameBasicType(newType, type))
         return LINK(type);
 
@@ -14148,7 +14214,7 @@ ITypeInfo * getSimplifiedType(ITypeInfo * type, bool isConditional, bool isSeria
 }
 
 
-static void simplifyRecordTypes(HqlExprArray & fields, IHqlExpression * cur, bool isConditional, bool & needsTransform, bool isKey, unsigned & count)
+static void simplifyFileViewRecordTypes(HqlExprArray & fields, IHqlExpression * cur, bool isConditional, bool & needsTransform, bool isKey, unsigned & count)
 {
     switch (cur->getOperator())
     {
@@ -14179,7 +14245,7 @@ static void simplifyRecordTypes(HqlExprArray & fields, IHqlExpression * cur, boo
                     bool childNeedsTransform = false;
                     IHqlExpression * fieldRecord = cur->queryRecord();
                     HqlExprArray subFields;
-                    simplifyRecordTypes(subFields, fieldRecord, isConditional, childNeedsTransform, isKey, count);
+                    simplifyFileViewRecordTypes(subFields, fieldRecord, isConditional, childNeedsTransform, isKey, count);
                     if (childNeedsTransform || hasLinkCountedModifier(type))
                     {
                         OwnedHqlExpr newRecord = fieldRecord->clone(subFields);
@@ -14200,14 +14266,14 @@ static void simplifyRecordTypes(HqlExprArray & fields, IHqlExpression * cur, boo
                     break;
                 }
             case type_set:
-                targetType.setown(getSimplifiedType(type, false, true));
+                targetType.setown(getSimplifiedType(type, false, true, diskAtom));
                 break;
             case type_row:
                 {
                     bool childNeedsTransform = false;
                     IHqlExpression * fieldRecord = cur->queryRecord();
                     HqlExprArray subFields;
-                    simplifyRecordTypes(subFields, fieldRecord, isConditional, childNeedsTransform, isKey, count);
+                    simplifyFileViewRecordTypes(subFields, fieldRecord, isConditional, childNeedsTransform, isKey, count);
                     if (childNeedsTransform)
                     {
                         OwnedHqlExpr newRecord = fieldRecord->clone(subFields);
@@ -14219,7 +14285,7 @@ static void simplifyRecordTypes(HqlExprArray & fields, IHqlExpression * cur, boo
                 }
                 return;
             default:
-                targetType.setown(getSimplifiedType(type, isConditional, true));
+                targetType.setown(getSimplifiedType(type, isConditional, true, diskAtom));
                 break;
             }
 
@@ -14245,7 +14311,7 @@ static void simplifyRecordTypes(HqlExprArray & fields, IHqlExpression * cur, boo
             StringBuffer name;
             name.append("unnamed").append(++count);
             subfields.append(*createField(createIdentifierAtom(name), makeBoolType(), NULL, createAttribute(__ifblockAtom)));
-            simplifyRecordTypes(subfields, cur->queryChild(1), true, needsTransform, isKey, count);
+            simplifyFileViewRecordTypes(subfields, cur->queryChild(1), true, needsTransform, isKey, count);
 //          fields.append(*createValue(no_ifblock, makeVoidType(), createValue(no_not, makeBoolType(), createConstant(false)), createRecord(subfields)));
             fields.append(*createValue(no_ifblock, makeNullType(), createConstant(true), createRecord(subfields)));
             break;
@@ -14258,11 +14324,11 @@ static void simplifyRecordTypes(HqlExprArray & fields, IHqlExpression * cur, boo
                 if (next->getOperator() == no_record)
                 {
                     HqlExprArray subfields;
-                    simplifyRecordTypes(subfields, next, isConditional, needsTransform, isKey, count);
+                    simplifyFileViewRecordTypes(subfields, next, isConditional, needsTransform, isKey, count);
                     fields.append(*cloneOrLink(cur, subfields));
                 }
                 else
-                    simplifyRecordTypes(fields, next, isConditional, needsTransform, isKey, count);
+                    simplifyFileViewRecordTypes(fields, next, isConditional, needsTransform, isKey, count);
             }
             break;
         }
@@ -14274,21 +14340,21 @@ static void simplifyRecordTypes(HqlExprArray & fields, IHqlExpression * cur, boo
     }
 }
 
-static IHqlExpression * simplifyRecordTypes(IHqlExpression * record, bool & needsTransform, bool isKey)
+static IHqlExpression * simplifyFileViewRecordTypes(IHqlExpression * record, bool & needsTransform, bool isKey)
 {
     assertex(record->getOperator() == no_record);
     HqlExprArray fields;
     unsigned count = 0;
-    simplifyRecordTypes(fields, record, false, needsTransform, isKey, count);
+    simplifyFileViewRecordTypes(fields, record, false, needsTransform, isKey, count);
     return cloneOrLink(record, fields);
 }
 
-extern HQL_API IHqlExpression * getSimplifiedRecord(IHqlExpression * record, bool isKey)
+extern HQL_API IHqlExpression * getFileViewerRecord(IHqlExpression * record, bool isKey)
 {
     bool needsTransform = false;
     try
     {
-        OwnedHqlExpr newRecord = simplifyRecordTypes(record, needsTransform, isKey);
+        OwnedHqlExpr newRecord = simplifyFileViewRecordTypes(record, needsTransform, isKey);
         if (needsTransform)
             return newRecord.getClear();
         return NULL;
@@ -14380,7 +14446,7 @@ extern HQL_API IHqlExpression * getSimplifiedTransform(IHqlExpression * tgt, IHq
 
 extern HQL_API bool isSimplifiedRecord(IHqlExpression * expr, bool isKey)
 {
-    OwnedHqlExpr simplified = getSimplifiedRecord(expr, isKey);
+    OwnedHqlExpr simplified = getFileViewerRecord(expr, isKey);
     return !simplified || (expr == simplified);
 }
 
@@ -15336,6 +15402,15 @@ bool recordTypesMatch(ITypeInfo * left, ITypeInfo * right)
     return false;
 }
 
+bool assertRecordTypesMatch(ITypeInfo * left, ITypeInfo * right)
+{
+    if (recordTypesMatch(left, right))
+        return true;
+
+    EclIR::dump_ir(left, right);
+    throwUnexpected();
+}
+
 
 bool recordTypesMatch(IHqlExpression * left, IHqlExpression * right)
 {

ecl/hql/hqlexpr.hpp

@@ -1473,8 +1473,8 @@ extern HQL_API IHqlExpression * queryTable(IHqlExpression * dataset);
 extern HQL_API node_operator queryTableMode(IHqlExpression * expr);
 
 // Code for producing simplified records that the file viewer can cope with
-extern HQL_API ITypeInfo * getSimplifiedType(ITypeInfo * type, bool isConditional, bool isSerialized);
-extern HQL_API IHqlExpression * getSimplifiedRecord(IHqlExpression * record, bool isKey);
+extern HQL_API ITypeInfo * getSimplifiedType(ITypeInfo * type, bool isConditional, bool isSerialized, _ATOM serialForm);
+extern HQL_API IHqlExpression * getFileViewerRecord(IHqlExpression * record, bool isKey);
 extern HQL_API IHqlExpression * getRecordMappingTransform(node_operator op, IHqlExpression * tgt, IHqlExpression * src, IHqlExpression * sourceSelector);
 extern HQL_API IHqlExpression * getSimplifiedTransform(IHqlExpression * tgt, IHqlExpression * src, IHqlExpression * sourceSelector);
 extern HQL_API IHqlExpression * removeVirtualAttributes(IHqlExpression * record);
@@ -1510,8 +1510,8 @@ extern HQL_API bool isActivityIndependentOfScope(IHqlExpression * expr);
 extern HQL_API bool exprReferencesDataset(IHqlExpression * expr, IHqlExpression * dataset);
 extern HQL_API bool canEvaluateInScope(const HqlExprCopyArray & activeScopes, IHqlExpression * expr);
 extern HQL_API bool canEvaluateInScope(const HqlExprCopyArray & activeScopes, const HqlExprCopyArray & required);
-extern HQL_API IHqlExpression * ensureDeserialized(IHqlExpression * expr, ITypeInfo * type);
-extern HQL_API IHqlExpression * ensureSerialized(IHqlExpression * expr);
+extern HQL_API IHqlExpression * ensureDeserialized(IHqlExpression * expr, ITypeInfo * type, _ATOM serialForm);
+extern HQL_API IHqlExpression * ensureSerialized(IHqlExpression * expr, _ATOM serialForm);
 extern HQL_API bool isDummySerializeDeserialize(IHqlExpression * expr);
 
 extern HQL_API unsigned getRepeatMax(IHqlExpression * expr);
@@ -1565,6 +1565,7 @@ extern HQL_API bool hasUninheritedAttribute(IHqlExpression * field);
 extern HQL_API IHqlExpression * extractChildren(IHqlExpression * value);
 extern HQL_API IHqlExpression * queryOnlyField(IHqlExpression * record);
 extern HQL_API bool recordTypesMatch(ITypeInfo * left, ITypeInfo * right);
+extern HQL_API bool assertRecordTypesMatch(ITypeInfo * left, ITypeInfo * right);
 extern HQL_API bool recordTypesMatch(IHqlExpression * left, IHqlExpression * right);
 extern HQL_API bool recordTypesMatchIgnorePayload(IHqlExpression *left, IHqlExpression *right);
 extern HQL_API IHqlExpression * queryOriginalRecord(IHqlExpression * expr);

ecl/hql/hqlexpr.ipp

@@ -1566,6 +1566,10 @@ public:
     ~CHqlDictionary();
 
     virtual IHqlExpression *clone(HqlExprArray &newkids);
+    virtual IHqlExpression *queryNormalizedSelector(bool skipIndex);
+    
+protected:
+    OwnedHqlExpr normalized;
 };
 
 

ecl/hql/hqlfold.cpp

@@ -3242,7 +3242,7 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
             if (child->isRecord())
             {
                 //Need to be careful to use the serialized record - otherwise record size can be inconsistent
-                OwnedHqlExpr record = getSerializedForm(child);
+                OwnedHqlExpr record = getSerializedForm(child, diskAtom);
                 if (expr->hasProperty(maxAtom))
                 {
                     if (maxRecordSizeCanBeDerived(record))

ecl/hql/hqlgram.hpp

@@ -473,6 +473,7 @@ public:
     IHqlExpression * createSortExpr(node_operator op, attribute & dsAttr, const attribute & orderAttr, HqlExprArray & args);
     IHqlExpression * createIffDataset(IHqlExpression * record, IHqlExpression * value);
 
+    bool isSingleValuedExpressionList(const attribute & attr);
     bool convertAllToAttribute(attribute &atr);
     IHqlExpression * convertToOutOfLineFunction(const ECLlocation & errpos, IHqlExpression  * expr);
 

ecl/hql/hqlgram.y

@@ -6254,6 +6254,9 @@ primexpr1
                         }
     | EXISTS '(' expressionList ')'
                         {
+                            if (parser->isSingleValuedExpressionList($3))
+                                parser->reportWarning(WRN_SILLY_EXISTS,$1.pos,"EXISTS() on a scalar expression is always true, was this intended?");
+
                             OwnedHqlExpr list = parser->createListFromExpressionList($3);
                             $$.setExpr(createValue(no_existslist, makeBoolType(), LINK(list)));
                             $$.setPosition($1);
@@ -10969,6 +10972,7 @@ sortItem
                             $$.setExpr(createValue(no_negate, makeVoidType(), e));
                             $$.setPosition($1);
                         }
+    | dictionary
     | FEW               {   $$.setExpr(createAttribute(fewAtom));   }
     | MANY              {   $$.setExpr(createAttribute(manyAtom));  }
     | MERGE             {   $$.setExpr(createAttribute(mergeAtom)); }

ecl/hql/hqlgram2.cpp

@@ -2358,6 +2358,8 @@ void HqlGram::addDatasetField(const attribute &errpos, _ATOM name, IHqlExpressio
     if (!name)
         name = createUnnamedFieldName();
     checkFieldnameValid(errpos, name);
+    if (value && !recordTypesMatch(record, value))
+        reportError(ERR_TYPEMISMATCH_DATASET, errpos, "Dataset expression has a different record from the field");
     if (queryPropertyInList(virtualAtom, attrs))
         reportError(ERR_BAD_FIELD_ATTR, errpos, "Virtual can only be specified on a scalar field");
     if (!attrs)
@@ -2375,6 +2377,8 @@ void HqlGram::addDictionaryField(const attribute &errpos, _ATOM name, IHqlExpres
     if (!name)
         name = createUnnamedFieldName();
     checkFieldnameValid(errpos, name);
+    if (value && !recordTypesMatch(record, value))
+        reportError(ERR_TYPEMISMATCH_DATASET, errpos, "Dataset expression has a different record from the field");
     if (queryPropertyInList(virtualAtom, attrs))
         reportError(ERR_BAD_FIELD_ATTR, errpos, "Virtual can only be specified on a scalar field");
     if (!attrs)
@@ -3839,6 +3843,16 @@ IHqlExpression * HqlGram::addDatasetSelector(IHqlExpression * lhs, IHqlExpressio
     return ret;
 }
 
+bool HqlGram::isSingleValuedExpressionList(const attribute & attr)
+{
+    IHqlExpression * expr = attr.queryExpr();
+    if (expr->getOperator() == no_comma)
+        return false;
+    if (expr->isList())
+        return false;
+    return true;
+}
+
 IHqlExpression * HqlGram::createListFromExprArray(const attribute & errpos, HqlExprArray & args)
 {
     ITypeInfo * retType = promoteToSameType(args, errpos, NULL, true);
@@ -5273,7 +5287,7 @@ void expandRecord(HqlExprArray & fields, IHqlExpression * selector, IHqlExpressi
     case no_field:
         {
             OwnedHqlExpr subSelector = createSelectExpr(LINK(selector), LINK(expr));
-            if (expr->queryRecord() && !expr->isDataset())
+            if (expr->queryRecord() && !expr->isDataset() && !expr->isDictionary())
                 expandRecord(fields, subSelector, expr->queryRecord());
             else
             {
@@ -6771,9 +6785,9 @@ void HqlGram::reportIndexFieldType(IHqlExpression * expr, bool isKeyed, const at
     StringBuffer s;
     getFriendlyTypeStr(expr, s);
     if (isKeyed)
-        reportError(ERR_INDEX_BADTYPE, errpos, "INDEX does not support keyed fields of type %s", s.str());
+        reportError(ERR_INDEX_BADTYPE, errpos, "INDEX does not currently support keyed fields of type '%s'", s.str());
     else
-        reportError(ERR_INDEX_BADTYPE, errpos, "INDEX does not support fields of type %s", s.str());
+        reportError(ERR_INDEX_BADTYPE, errpos, "INDEX does not currently support fields of type '%s'", s.str());
 }
 
 void HqlGram::checkIndexFieldType(IHqlExpression * expr, bool isPayload, bool insideNestedRecord, const attribute & errpos)

ecl/hql/hqlpmap.cpp

@@ -1137,15 +1137,19 @@ void RecordTransformCreator::createAssignments(HqlExprArray & assigns, IHqlExpre
 
             ITypeInfo * sourceType = source->queryType();
             ITypeInfo * targetType = target->queryType();
+            type_t tc = targetType->getTypeCode();
+
+            if (source->isDictionary())
+                source.setown(createDataset(no_datasetfromdictionary, source.getClear()));
+
             if (!recordTypesMatch(sourceType, targetType))
             {
-                type_t tc = sourceType->getTypeCode();
                 if (tc == type_record || tc == type_row)
                 {
                     OwnedHqlExpr tform = createMappingTransform(no_transform, target->queryRecord(), source);
                     source.setown(createRow(no_createrow, tform.getClear()));
                 }
-                else if ((tc == type_table) || (tc == type_groupedtable))
+                else if ((tc == type_table) || (tc == type_groupedtable) || (tc == type_dictionary))
                 {
                     //self.target := project(srcSelect, transform(...));
                     OwnedHqlExpr seq = createSelectorSequence();
@@ -1155,6 +1159,9 @@ void RecordTransformCreator::createAssignments(HqlExprArray & assigns, IHqlExpre
                 }
             }
 
+            if (tc == type_dictionary)
+                source.setown(createDictionary(no_createdictionary, source.getClear()));
+
             assigns.append(*createAssign(LINK(target), LINK(source)));
             break;
         }
@@ -1193,14 +1200,14 @@ IHqlExpression * createRecordMappingTransform(node_operator op, IHqlExpression *
 }
 
 
-IHqlExpression * replaceMemorySelectorWithSerializedSelector(IHqlExpression * expr, IHqlExpression * memoryRecord, node_operator side, IHqlExpression * selSeq)
+IHqlExpression * replaceMemorySelectorWithSerializedSelector(IHqlExpression * expr, IHqlExpression * memoryRecord, node_operator side, IHqlExpression * selSeq, _ATOM serializeVariety)
 {
     if (!expr) 
         return NULL;
 
     assertex(side);
     assertex(memoryRecord->isRecord());
-    OwnedHqlExpr serializedRecord = getSerializedForm(memoryRecord);
+    OwnedHqlExpr serializedRecord = getSerializedForm(memoryRecord, serializeVariety);
     if (memoryRecord == serializedRecord)
         return LINK(expr);
 

ecl/hql/hqlpmap.hpp

@@ -160,7 +160,7 @@ protected:
 
 
 extern HQL_API IHqlExpression * createRecordMappingTransform(node_operator op, IHqlExpression * targetRecord, IHqlExpression * sourceSelector);
-extern HQL_API IHqlExpression * replaceMemorySelectorWithSerializedSelector(IHqlExpression * expr, IHqlExpression * memoryRecord, node_operator side, IHqlExpression * selSeq);
+extern HQL_API IHqlExpression * replaceMemorySelectorWithSerializedSelector(IHqlExpression * expr, IHqlExpression * memoryRecord, node_operator side, IHqlExpression * selSeq, _ATOM serializeVariety);
 
 extern HQL_API TableProjectMapper * createProjectMapper(IHqlExpression * expr);
 extern HQL_API TableProjectMapper * createProjectMapper(IHqlExpression * mapping, IHqlExpression * parent);

ecl/hql/hqltrans.cpp

@@ -616,6 +616,7 @@ ITypeInfo * HqlTransformerBase::transformType(ITypeInfo * type)
     case type_token:
     case type_groupedtable:
     case type_table:
+    case type_dictionary:
         {
             ITypeInfo * childType = type->queryChildType();
             OwnedITypeInfo newChildType = safeTransformType(childType);

ecl/hql/hqlutil.cpp

@@ -54,7 +54,6 @@ static IHqlExpression * cacheEmbeddedAttr;
 static IHqlExpression * cacheInlineAttr;
 static IHqlExpression * cacheLinkCountedAttr;
 static IHqlExpression * cacheReferenceAttr;
-static IHqlExpression * cacheSerializedFormAttr;
 static IHqlExpression * cacheStreamedAttr;
 static IHqlExpression * cacheUnadornedAttr;
 static IHqlExpression * matchxxxPseudoFile;
@@ -83,7 +82,6 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     cacheInlineAttr = createAttribute(inlineAtom);
     cacheLinkCountedAttr = createAttribute(_linkCounted_Atom);
     cacheReferenceAttr = createAttribute(referenceAtom);
-    cacheSerializedFormAttr = createAttribute(_attrSerializedForm_Atom);
     cacheStreamedAttr = createAttribute(streamedAtom);
     cacheUnadornedAttr = createAttribute(_attrUnadorned_Atom);
     matchxxxPseudoFile = createDataset(no_pseudods, createRecord()->closeExpr(), createAttribute(matchxxxPseudoFileAtom));
@@ -110,7 +108,6 @@ MODULE_EXIT()
     cacheInlineAttr->Release();
     cacheLinkCountedAttr->Release();
     cacheReferenceAttr->Release();
-    cacheSerializedFormAttr->Release();
     cacheStreamedAttr->Release();
     cacheUnadornedAttr->Release();
     matchxxxPseudoFile->Release();
@@ -197,11 +194,6 @@ IHqlExpression * queryRequiresDestructorAttr(bool value)
 }
 #endif
 
-IHqlExpression * querySerializedFormAttr()
-{
-    return cacheSerializedFormAttr;
-}
-
 IHqlExpression * queryUnadornedAttr()
 {
     return cacheUnadornedAttr;
@@ -5275,10 +5267,14 @@ void TempTableTransformer::createTempTableAssign(HqlExprArray & assigns, IHqlExp
                             castValue.setown(createRow(no_createrow, LINK(transform)));
                         }
                     }
+                    if (target->isDictionary() && !castValue->isDictionary())
+                        castValue.setown(createDictionary(no_createdictionary, castValue.getClear()));
                 }
                 else
                 {
-                    if (expr->isDataset())
+                    if (expr->isDictionary())
+                        castValue.setown(createDictionary(no_null, LINK(record)));
+                    else if (expr->isDataset())
                         castValue.setown(createDataset(no_null, LINK(record)));
                     else
                         castValue.setown(createRow(no_null, LINK(record)));

ecl/hql/hqlutil.hpp

@@ -576,7 +576,6 @@ protected:
 
 extern HQL_API IHqlExpression * queryDefaultMaxRecordLengthExpr();
 extern HQL_API IHqlExpression * getFixedSizeAttr(unsigned size);
-extern HQL_API IHqlExpression * querySerializedFormAttr();
 extern HQL_API IHqlExpression * queryAlignedAttr();
 extern HQL_API IHqlExpression * queryLinkCountedAttr();
 extern HQL_API IHqlExpression * queryUnadornedAttr();

ecl/hqlcpp/hqlcatom.cpp

@@ -207,9 +207,13 @@ _ATOM deleteFileAtom;
 _ATOM dependencyAtom;
 _ATOM deserializerAtom;
 _ATOM deserializeRawAtom;
+_ATOM deserializeChildDictionaryFromDatasetFromStreamAtom;
+_ATOM deserializeChildDictionaryFromStreamAtom;
+_ATOM deserializeChildRowsetFromStreamAtom;
 _ATOM deserializeCStringXAtom;
 _ATOM deserializeDataXAtom;
 _ATOM deserializeDatasetXAtom;
+_ATOM deserializeDictionaryXAtom;
 _ATOM deserializeGroupedDatasetXAtom;
 _ATOM deserializeQStrXAtom;
 _ATOM deserializeRowAtom;
@@ -220,7 +224,6 @@ _ATOM deserializeStringXAtom;
 _ATOM deserializeUnicodeXAtom;
 _ATOM deserializeUtf8XAtom;
 _ATOM deserializeVUnicodeXAtom;
-_ATOM deserializerRowsetHelperAtom;
 _ATOM deserializerReadNAtom;
 _ATOM deserializerReadPackedIntAtom;
 _ATOM deserializerReadSizeAtom;
@@ -330,6 +333,7 @@ _ATOM getWorkflowIdAtom;
 _ATOM getWuidAtom;
 _ATOM goAtom;
 _ATOM groupedDataset2RowsetXAtom;
+_ATOM groupedRowset2DatasetXAtom;
 _ATOM guardAtom;
 _ATOM hash32DataAtom;
 _ATOM hash32Data1Atom;
@@ -515,13 +519,16 @@ _ATOM roundToAtom;
 _ATOM roundupAtom;
 _ATOM rowAllocatorMarkerAtom;
 _ATOM rowset2DatasetXAtom;
+_ATOM rtlDeserializeDictionaryAtom;
+_ATOM rtlDeserializeDictionaryFromDatasetAtom;
 _ATOM rtlDeserializeRowAtom;
 _ATOM rtlDeserializeToBuilderAtom;
 _ATOM rtlFieldKeyMarkerAtom;
 _ATOM rtlMaxAtom;
 _ATOM rtlMinAtom;
 _ATOM rtlRandomAtom;
-_ATOM rtlSerializeRowAtom;
+_ATOM rtlSerializeDictionaryAtom;
+_ATOM rtlSerializeDictionaryToDatasetAtom;
 _ATOM rtlSerializeToBuilderAtom;
 _ATOM searchTableInteger4Atom;
 _ATOM searchTableInteger8Atom;
@@ -533,12 +540,16 @@ _ATOM searchVUnicodeTableAtom;
 _ATOM selectClusterAtom;
 _ATOM _selfJoinPlaceholder_Atom;
 _ATOM serializerAtom;
+_ATOM serializeChildDictionaryToStreamAtom;
+_ATOM serializeChildDictionaryToDatasetToStreamAtom;
+_ATOM serializeChildRowsetToStreamAtom;
 _ATOM serializeCStringXAtom;
 _ATOM serializeDataXAtom;
 _ATOM serializeDatasetXAtom;
 _ATOM serializeGroupedDatasetXAtom;
 _ATOM serializeStringXAtom;
 _ATOM serializeBoolAtom;
+_ATOM serializeDictionaryXAtom;
 _ATOM serializeFixedDataAtom;
 _ATOM serializeFixedStringAtom;
 _ATOM serializeLPDataAtom;
@@ -555,7 +566,6 @@ _ATOM serializeSetAtom;
 _ATOM serializeUnicodeXAtom;
 _ATOM serializeUtf8XAtom;
 _ATOM serializerPutAtom;
-_ATOM serializerRowsetHelperAtom;
 _ATOM serializerBeginNestedAtom;
 _ATOM serializerEndNestedAtom;
 _ATOM serializerInstanceMarkerAtom;
@@ -918,9 +928,13 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(dependency);
     MAKEATOM(deserializer);
     MAKEATOM(deserializeRaw);
+    MAKEATOM(deserializeChildDictionaryFromStream);
+    MAKEATOM(deserializeChildDictionaryFromDatasetFromStream);
+    MAKEATOM(deserializeChildRowsetFromStream);
     MAKEATOM(deserializeCStringX);
     MAKEATOM(deserializeDataX);
     MAKEATOM(deserializeDatasetX);
+    MAKEATOM(deserializeDictionaryX);
     MAKEATOM(deserializeGroupedDatasetX);
     MAKEATOM(deserializeQStrX);
     MAKEATOM(deserializeRow);
@@ -931,7 +945,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(deserializeUnicodeX);
     MAKEATOM(deserializeUtf8X);
     MAKEATOM(deserializeVUnicodeX);
-    MAKEATOM(deserializerRowsetHelper);
     MAKEATOM(deserializerReadN);
     MAKEATOM(deserializerReadPackedInt);
     MAKEATOM(deserializerReadSize);
@@ -1040,6 +1053,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(getWorkflowId);
     MAKEATOM(getWuid);
     MAKEATOM(groupedDataset2RowsetX);
+    MAKEATOM(groupedRowset2DatasetX);
     MAKEATOM(go);
     MAKEATOM(guard);
     MAKEATOM(hash32Data);
@@ -1254,13 +1268,16 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     roundupAtom = createAtom("_roundup");
     MAKEATOM(rowAllocatorMarker);
     MAKEATOM(rowset2DatasetX);
+    MAKEATOM(rtlDeserializeDictionary);
+    MAKEATOM(rtlDeserializeDictionaryFromDataset);
     MAKEATOM(rtlDeserializeRow);
     MAKEATOM(rtlDeserializeToBuilder);
     MAKEATOM(rtlFieldKeyMarker);
     MAKEATOM(rtlMax);
     MAKEATOM(rtlMin);
     MAKEATOM(rtlRandom);
-    MAKEATOM(rtlSerializeRow);
+    MAKEATOM(rtlSerializeDictionary);
+    MAKEATOM(rtlSerializeDictionaryToDataset);
     MAKEATOM(rtlSerializeToBuilder);
     MAKEATOM(searchTableInteger4);
     MAKEATOM(searchTableInteger8);
@@ -1272,14 +1289,19 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(selectCluster);
     MAKESYSATOM(selfJoinPlaceholder);
     MAKEATOM(serializer);
+    MAKEATOM(serializeBool);
+    MAKEATOM(serializeChildDictionaryToStream);
+    MAKEATOM(serializeChildDictionaryToDatasetToStream);
+    MAKEATOM(serializeChildRowsetToStream);
     MAKEATOM(serializeCStringX);
     MAKEATOM(serializeDataX);
     MAKEATOM(serializeDatasetX);
-    MAKEATOM(serializeGroupedDatasetX);
     MAKEATOM(serializeStringX);
-    MAKEATOM(serializeBool);
+    MAKEATOM(serializeDictionaryX);
     MAKEATOM(serializeFixedData);
     MAKEATOM(serializeFixedString);
+    MAKEATOM(serializeGroupedDatasetX);
+    MAKEATOM(serializeGroupedRowsetX);
     MAKEATOM(serializeLPData);
     MAKEATOM(serializeLPQString);
     MAKEATOM(serializeLPString);
@@ -1289,11 +1311,9 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(serializeReal8);
     MAKEATOM(serializeRow);
     MAKEATOM(serializeRowsetX);
-    MAKEATOM(serializeGroupedRowsetX);
     MAKEATOM(serializeSet);
     MAKEATOM(serializeUnicodeX);
     MAKEATOM(serializeUtf8X);
-    MAKEATOM(serializerRowsetHelper);
     MAKEATOM(serializerPut);
     MAKEATOM(serializerBeginNested);
     MAKEATOM(serializerEndNested);

ecl/hqlcpp/hqlcatom.hpp

@@ -207,9 +207,13 @@ extern _ATOM deleteFileAtom;
 extern _ATOM dependencyAtom;
 extern _ATOM deserializerAtom;
 extern _ATOM deserializeRawAtom;
+extern _ATOM deserializeChildDictionaryFromDatasetFromStreamAtom;
+extern _ATOM deserializeChildDictionaryFromStreamAtom;
+extern _ATOM deserializeChildRowsetFromStreamAtom;
 extern _ATOM deserializeCStringXAtom;
 extern _ATOM deserializeDataXAtom;
 extern _ATOM deserializeDatasetXAtom;
+extern _ATOM deserializeDictionaryXAtom;
 extern _ATOM deserializeGroupedDatasetXAtom;
 extern _ATOM deserializeQStrXAtom;
 extern _ATOM deserializeRowAtom;
@@ -220,7 +224,6 @@ extern _ATOM deserializeSetAtom;
 extern _ATOM deserializeUnicodeXAtom;
 extern _ATOM deserializeUtf8XAtom;
 extern _ATOM deserializeVUnicodeXAtom;
-extern _ATOM deserializerRowsetHelperAtom;
 extern _ATOM deserializerReadNAtom;
 extern _ATOM deserializerReadPackedIntAtom;
 extern _ATOM deserializerReadSizeAtom;
@@ -329,6 +332,7 @@ extern _ATOM getRootResultAtom;
 extern _ATOM getWorkflowIdAtom;
 extern _ATOM getWuidAtom;
 extern _ATOM groupedDataset2RowsetXAtom;
+extern _ATOM groupedRowset2DatasetXAtom;
 extern _ATOM goAtom;
 extern _ATOM guardAtom;
 extern _ATOM hash32DataAtom;
@@ -515,13 +519,16 @@ extern _ATOM roundToAtom;
 extern _ATOM roundupAtom;
 extern _ATOM rowAllocatorMarkerAtom;
 extern _ATOM rowset2DatasetXAtom;
+extern _ATOM rtlDeserializeDictionaryAtom;
+extern _ATOM rtlDeserializeDictionaryFromDatasetAtom;
 extern _ATOM rtlDeserializeRowAtom;
 extern _ATOM rtlDeserializeToBuilderAtom;
 extern _ATOM rtlFieldKeyMarkerAtom;
 extern _ATOM rtlMaxAtom;
 extern _ATOM rtlMinAtom;
 extern _ATOM rtlRandomAtom;
-extern _ATOM rtlSerializeRowAtom;
+extern _ATOM rtlSerializeDictionaryAtom;
+extern _ATOM rtlSerializeDictionaryToDatasetAtom;
 extern _ATOM rtlSerializeToBuilderAtom;
 extern _ATOM searchTableInteger4Atom;
 extern _ATOM searchTableInteger8Atom;
@@ -533,12 +540,16 @@ extern _ATOM searchVUnicodeTableAtom;
 extern _ATOM selectClusterAtom;
 extern _ATOM _selfJoinPlaceholder_Atom;
 extern _ATOM serializerAtom;
+extern _ATOM serializeChildDictionaryToStreamAtom;
+extern _ATOM serializeChildDictionaryToDatasetToStreamAtom;
+extern _ATOM serializeChildRowsetToStreamAtom;
 extern _ATOM serializeCStringXAtom;
 extern _ATOM serializeDataXAtom;
 extern _ATOM serializeDatasetXAtom;
 extern _ATOM serializeGroupedDatasetXAtom;
 extern _ATOM serializeStringXAtom;
 extern _ATOM serializeBoolAtom;
+extern _ATOM serializeDictionaryXAtom;
 extern _ATOM serializeFixedDataAtom;
 extern _ATOM serializeFixedStringAtom;
 extern _ATOM serializeQStrXAtom;
@@ -554,7 +565,6 @@ extern _ATOM serializeGroupedRowsetXAtom;
 extern _ATOM serializeSetAtom;
 extern _ATOM serializeUnicodeXAtom;
 extern _ATOM serializeUtf8XAtom;
-extern _ATOM serializerRowsetHelperAtom;
 extern _ATOM serializerPutAtom;
 extern _ATOM serializerBeginNestedAtom;
 extern _ATOM serializerEndNestedAtom;

ecl/hqlcpp/hqlckey.cpp

@@ -624,7 +624,7 @@ void KeyedJoinInfo::buildTransformBody(BuildCtx & ctx, IHqlExpression * transfor
     IHqlExpression * rowsid = expr->queryProperty(_rowsid_Atom);
 
     OwnedHqlExpr originalRight = createSelector(no_right, rhsRecord, joinSeq);
-    OwnedHqlExpr serializedRhsRecord = getSerializedForm(rhsRecord);
+    OwnedHqlExpr serializedRhsRecord = getSerializedForm(rhsRecord, diskAtom);
     OwnedHqlExpr serializedRight = createSelector(no_right, serializedRhsRecord, joinSeq);
 
     OwnedHqlExpr joinDataset = createDataset(no_anon, LINK(extractJoinFieldsRecord));
@@ -644,12 +644,12 @@ void KeyedJoinInfo::buildTransformBody(BuildCtx & ctx, IHqlExpression * transfor
             //References to ROWS(originalRight) need to be replaced with DESERIALIZE(ROWS(serializedRight))
             OwnedHqlExpr originalRows = createDataset(no_rows, LINK(originalRight), LINK(rowsid));
             OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(extractedRight), LINK(rowsid));
-            OwnedHqlExpr deserializedRows = ensureDeserialized(rowsExpr, rhsRecord->queryType());
+            OwnedHqlExpr deserializedRows = ensureDeserialized(rowsExpr, rhsRecord->queryType(), diskAtom);
             newTransform.setown(replaceExpression(newTransform, originalRows, deserializedRows));
         }
     }
 
-    newTransform.setown(replaceMemorySelectorWithSerializedSelector(newTransform, rhsRecord, no_right, joinSeq));
+    newTransform.setown(replaceMemorySelectorWithSerializedSelector(newTransform, rhsRecord, no_right, joinSeq, diskAtom));
 
     OwnedHqlExpr fileposExpr = getFilepos(extractedRight, false);
     if (extractJoinFieldsTransform)
@@ -770,9 +770,9 @@ IHqlExpression * KeyedJoinInfo::optimizeTransfer(HqlExprArray & fields, HqlExprA
                 }
 
                 IHqlExpression * matchField = &fields.item(match);
-                OwnedHqlExpr serializedField = getSerializedForm(matchField);
+                OwnedHqlExpr serializedField = getSerializedForm(matchField, diskAtom);
                 OwnedHqlExpr result = createSelectExpr(getActiveTableSelector(), LINK(serializedField));
-                return ensureDeserialized(result, matchField->queryType());
+                return ensureDeserialized(result, matchField->queryType(), diskAtom);
             }
             break;
         }
@@ -829,7 +829,7 @@ void KeyedJoinInfo::optimizeTransfer(SharedHqlExpr & targetDataset, SharedHqlExp
             if (newFilter && (newExtraFilter || !hasExtra) && fields.ordinality() < getFieldCount(dataset->queryRecord()))
             {
                 OwnedHqlExpr extractedRecord = translator.createRecordInheritMaxLength(fields, dataset);
-                OwnedHqlExpr serializedRecord = getSerializedForm(extractedRecord);
+                OwnedHqlExpr serializedRecord = getSerializedForm(extractedRecord, diskAtom);
                 targetDataset.setown(createDataset(no_anon, LINK(serializedRecord), NULL));
 
                 HqlExprArray assigns;
@@ -837,8 +837,8 @@ void KeyedJoinInfo::optimizeTransfer(SharedHqlExpr & targetDataset, SharedHqlExp
                 ForEachItemIn(i, fields)
                 {
                     IHqlExpression * curField = &fields.item(i);
-                    OwnedHqlExpr serializedField = getSerializedForm(curField);
-                    OwnedHqlExpr value = ensureSerialized(&values.item(i));
+                    OwnedHqlExpr serializedField = getSerializedForm(curField, diskAtom);
+                    OwnedHqlExpr value = ensureSerialized(&values.item(i), diskAtom);
                     assigns.append(*createAssign(createSelectExpr(LINK(self), LINK(serializedField)), LINK(value)));
                 }
                 targetTransform.setown(createValue(no_newtransform, makeTransformType(serializedRecord->getType()), assigns));
@@ -849,15 +849,15 @@ void KeyedJoinInfo::optimizeTransfer(SharedHqlExpr & targetDataset, SharedHqlExp
                     extraFilter->setown(replaceSelector(newExtraFilter, queryActiveTableSelector(), leftSelect));
 
             }
-            else if (recordRequiresSerialization(record))
+            else if (recordRequiresSerialization(record, diskAtom))
             {
-                OwnedHqlExpr serializedRecord = getSerializedForm(record);
+                OwnedHqlExpr serializedRecord = getSerializedForm(record, diskAtom);
                 targetDataset.setown(createDataset(no_anon, LINK(serializedRecord)));
                 targetTransform.setown(createRecordMappingTransform(no_transform, serializedRecord, oldLeft));
 
-                filter.setown(replaceMemorySelectorWithSerializedSelector(filter, record, no_left, joinSeq));
+                filter.setown(replaceMemorySelectorWithSerializedSelector(filter, record, no_left, joinSeq, diskAtom));
                 if (hasExtra)
-                    extraFilter->setown(replaceMemorySelectorWithSerializedSelector(*extraFilter, record, no_left, joinSeq));
+                    extraFilter->setown(replaceMemorySelectorWithSerializedSelector(*extraFilter, record, no_left, joinSeq, diskAtom));
             }
             else
                 targetDataset.set(dataset);
@@ -991,7 +991,7 @@ void KeyedJoinInfo::optimizeExtractJoinFields()
                 doExtract = true;
         }
 
-        if (recordRequiresSerialization(rightRecord))
+        if (recordRequiresSerialization(rightRecord, diskAtom))
             doExtract = true;
     }
 
@@ -1004,7 +1004,7 @@ void KeyedJoinInfo::optimizeExtractJoinFields()
         {
             if (!extractedRecord)
                 extractedRecord.setown(translator.createRecordInheritMaxLength(fieldsAccessed, rawRhs));
-            extractJoinFieldsRecord.setown(getSerializedForm(extractedRecord));
+            extractJoinFieldsRecord.setown(getSerializedForm(extractedRecord, diskAtom));
             OwnedHqlExpr self = getSelf(extractJoinFieldsRecord);
             OwnedHqlExpr memorySelf = getSelf(extractedRecord);
 
@@ -1040,7 +1040,7 @@ void KeyedJoinInfo::optimizeExtractJoinFields()
                         OwnedHqlExpr tgt = createSelectExpr(LINK(self), LINK(curSerializedField));
                         OwnedHqlExpr src = createSelectExpr(LINK(memorySelf), LINK(curMemoryField));
                         OwnedHqlExpr mappedSrc = fieldMapper.expandFields(src, memorySelf, left, rawRhs);
-                        assigns.append(*createAssign(tgt.getClear(), ensureSerialized(mappedSrc)));
+                        assigns.append(*createAssign(tgt.getClear(), ensureSerialized(mappedSrc, diskAtom)));
                     }
                 }
             }

ecl/hqlcpp/hqlcpp.cpp

@@ -4120,7 +4120,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
     case type_table:
     case type_groupedtable:
         {
-            if (recordRequiresSerialization(::queryRecord(exprType)) || hasLinkCountedModifier(_exprType))
+            if (recordRequiresLinkCount(::queryRecord(exprType)) || hasLinkCountedModifier(_exprType))
             {
                 assertex(format != FormatBlockedDataset);
                 format = FormatLinkedDataset;
@@ -8066,6 +8066,13 @@ void HqlCppTranslator::doBuildAssignCompareElement(BuildCtx & ctx, EvaluateCompa
     switch (tc)
     {
     case type_dictionary:
+        {
+            //You can't iterate dictionaries, so convert to datasets first.  A bit of a silly comparison anyway
+            OwnedHqlExpr dsLeft = createDataset(no_datasetfromdictionary, LINK(left));
+            OwnedHqlExpr dsRight = createDataset(no_datasetfromdictionary, LINK(right));
+            doBuildAssignCompareTable(ctx, info, dsLeft, dsRight);
+            return;
+        }
     case type_table:
     case type_groupedtable:
         doBuildAssignCompareTable(ctx, info, left, right);
@@ -8829,7 +8836,7 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
             }
             else
             {
-                OwnedHqlExpr serialized = ::ensureSerialized(elem);
+                OwnedHqlExpr serialized = ::ensureSerialized(elem, diskAtom);
                 buildDataset(ctx, serialized, bound, FormatBlockedDataset);
                 length.setown(getBoundSize(bound));
                 ptr.setown(getPointer(bound.expr));

ecl/hqlcpp/hqlcpp.ipp

@@ -197,7 +197,7 @@ interface IHqlCppDatasetBuilder : public IInterface
 public:
     virtual void buildDeclare(BuildCtx & ctx) = 0;
     virtual BoundRow * buildCreateRow(BuildCtx & ctx) = 0;                  //NB: Must be called within a addGroup(), or another un-ambiguous child context, may create a filter
-    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput) = 0;
+    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput, _ATOM serializeForm) = 0;
     virtual void finishRow(BuildCtx & ctx, BoundRow * selfCursor) = 0;
     virtual void buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target) = 0;
     virtual void buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound) = 0;
@@ -315,8 +315,8 @@ public:
     virtual void set(BuildCtx & ctx, IHqlExpression * expr) = 0;
     virtual void setRow(BuildCtx & ctx, IReferenceSelector * rhs) = 0;
 
-    virtual void buildDeserialize(BuildCtx & ctx, IHqlExpression * helper) = 0;
-    virtual void buildSerialize(BuildCtx & ctx, IHqlExpression * helper) = 0;
+    virtual void buildDeserialize(BuildCtx & ctx, IHqlExpression * helper, _ATOM serializeForm) = 0;
+    virtual void buildSerialize(BuildCtx & ctx, IHqlExpression * helper, _ATOM serializeForm) = 0;
 
 //managing the selection
     virtual AColumnInfo * queryColumn() = 0;
@@ -1008,9 +1008,9 @@ public:
     IHqlExpression * getRecordSize(IHqlExpression * dataset);
     unsigned getCsvMaxLength(IHqlExpression * csvAttr);
     IHqlExpression * createRecordInheritMaxLength(HqlExprArray & fields, IHqlExpression * donor);
-    void ensureRowSerializer(StringBuffer & serializerName, BuildCtx & ctx, IHqlExpression * record, _ATOM kind);
+    void ensureRowSerializer(StringBuffer & serializerName, BuildCtx & ctx, IHqlExpression * record, _ATOM format, _ATOM kind);
     void ensureRowPrefetcher(StringBuffer & prefetcherName, BuildCtx & ctx, IHqlExpression * record);
-    IHqlExpression * createRowSerializer(BuildCtx & ctx, IHqlExpression * record, _ATOM kind);
+    IHqlExpression * createSerializer(BuildCtx & ctx, IHqlExpression * record, _ATOM format, _ATOM kind);
 
     AliasKind buildExprInCorrectContext(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, bool evaluateLocally);
     ParentExtract * createExtractBuilder(BuildCtx & ctx, PEtype type, IHqlExpression * graphId, IHqlExpression * expr, bool doDeclare);
@@ -1175,10 +1175,10 @@ public:
     void doBuildInCaseInfo(IHqlExpression * expr, HqlCppCaseInfo & info, IHqlExpression * normalizedValues = NULL);
     void doBuildAssignToFromUnicode(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
 
-    void buildAssignLinkedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
-    void buildAssignSerializedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
-    void buildLinkedDataset(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * expr, CHqlBoundExpr & tgt);
-    void buildSerializedDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
+    void buildAssignDeserializedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, _ATOM serializeForm);
+    void buildAssignSerializedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, _ATOM serializeForm);
+    void buildDeserializedDataset(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * expr, CHqlBoundExpr & tgt, _ATOM serializeForm);
+    void buildSerializedDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, _ATOM serializeForm);
 
     void buildDatasetAssignAggregate(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr);
     void buildDatasetAssignChoose(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr);
@@ -1585,7 +1585,7 @@ public:
     void buildConnectOrders(BuildCtx & ctx, ABoundActivity * slaveActivity, ABoundActivity * masterActivity);
     void buildDedupFilterFunction(BuildCtx & ctx, HqlExprArray & equalities, HqlExprArray & conds, IHqlExpression * dataset, IHqlExpression * selSeq);
     void buildDedupSerializeFunction(BuildCtx & ctx, const char * funcName, IHqlExpression * srcDataset, IHqlExpression * tgtDataset, HqlExprArray & srcValues, HqlExprArray & tgtValues, IHqlExpression * selSeq);
-    void buildDictionaryHashClass(IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &lookupHelperName);
+    void buildDictionaryHashClass(IHqlExpression *record, StringBuffer &lookupHelperName);
     void buildDictionaryHashMember(BuildCtx & ctx, IHqlExpression *dictionary, const char * memberName);
     void buildHashClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, const DatasetReference & dataset);
     void buildHashOfExprsClass(BuildCtx & ctx, const char * name, IHqlExpression * cond, const DatasetReference & dataset, bool compareToSelf);
@@ -1645,8 +1645,8 @@ public:
     void buildSetXmlSerializer(StringBuffer & helper, ITypeInfo * valueType);
 
     void buildMetaMember(BuildCtx & ctx, IHqlExpression * datasetOrRecord, bool isGrouped, const char * name);
-    void buildMetaSerializerClass(BuildCtx & ctx, IHqlExpression * record, const char * serializerName);
-    void buildMetaDeserializerClass(BuildCtx & ctx, IHqlExpression * record, const char * deserializerName);
+    void buildMetaSerializerClass(BuildCtx & ctx, IHqlExpression * record, const char * serializerName, _ATOM serializeForm);
+    void buildMetaDeserializerClass(BuildCtx & ctx, IHqlExpression * record, const char * deserializerName, _ATOM serializeForm);
     bool buildMetaPrefetcherClass(BuildCtx & ctx, IHqlExpression * record, const char * prefetcherName);
 
     void buildLibraryInstanceExtract(BuildCtx & ctx, HqlCppLibraryInstance * libraryInstance);
@@ -1670,7 +1670,7 @@ public:
     void buildXmlReadTransform(IHqlExpression * dataset, StringBuffer & className, bool & usesContents);
     void doBuildXmlReadMember(ActivityInstance & instance, IHqlExpression * expr, const char * functionName, bool & usesContents);
 
-    void ensureSerialized(const CHqlBoundTarget & variable, BuildCtx & serializectx, BuildCtx & deserializectx, const char * inBufferName, const char * outBufferName);
+    void ensureSerialized(const CHqlBoundTarget & variable, BuildCtx & serializectx, BuildCtx & deserializectx, const char * inBufferName, const char * outBufferName, _ATOM serializeForm);
     void ensureRowAllocator(StringBuffer & allocatorName, BuildCtx & ctx, IHqlExpression * record, IHqlExpression * activityId);
     IHqlExpression * createRowAllocator(BuildCtx & ctx, IHqlExpression * record);
 
@@ -1710,7 +1710,7 @@ public:
     void buildLimitHelpers(BuildCtx & ctx, IHqlExpression * rowLimit, IHqlExpression * failAction, bool isSkip, IHqlExpression * filename, unique_id_t id);
 
     void doGenerateMetaDestruct(BuildCtx & ctx, IHqlExpression * selector, IHqlExpression * record);
-    void generateMetaRecordSerialize(BuildCtx & ctx, IHqlExpression * dataset, const char * serializerName, const char * deserializerName, const char * prefetcherName);
+    void generateMetaRecordSerialize(BuildCtx & ctx, IHqlExpression * record, const char * diskSerializerName, const char * diskDeserializerName, const char * internalSerializerName, const char * internalDeserializerName, const char * prefetcherName);
 
     void filterExpandAssignments(BuildCtx & ctx, TransformBuilder * builder, HqlExprArray & assigns, IHqlExpression * expr);
 
@@ -1757,8 +1757,8 @@ protected:
 
     void doFilterAssignment(BuildCtx & ctx, TransformBuilder * builder, HqlExprArray & assigns, IHqlExpression * expr);
     void doFilterAssignments(BuildCtx & ctx, TransformBuilder * builder, HqlExprArray & assigns, IHqlExpression * expr);
-    void generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear);
-    void generateSerializeFunction(BuildCtx & ctx, const char * funcName, bool serialize, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects);
+    void generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear, node_operator serializeOp, _ATOM serialForm);
+    void generateSerializeFunction(BuildCtx & ctx, const char * funcName, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, node_operator serializeOp, _ATOM serialForm);
     void generateSerializeKey(BuildCtx & ctx, node_operator side, const DatasetReference & dataset, HqlExprArray & sorts, bool isGlobal, bool generateCompares, bool canReuseLeft);             //NB: sorts are ats.xyz
     void generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx, node_operator index, const DatasetReference & dataset, HqlExprArray & sorts, bool canRemoveSort, IHqlExpression * noSortAttr, bool canReuseLeft, bool isLightweight, bool isLocal);
     void addSchemaField(IHqlExpression *field, MemoryBuffer &schema, IHqlExpression *selector);

ecl/hqlcpp/hqlcppc.hpp

@@ -51,8 +51,8 @@ public:
     virtual bool modifyColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value, node_operator op) = 0;
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value) = 0;
     virtual void setRow(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IReferenceSelector * value) = 0;
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper) = 0;
-    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper) = 0;
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm) = 0;
+    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm) = 0;
     virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state) = 0;
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state) = 0;
 };

ecl/hqlcpp/hqlcppds.cpp

@@ -119,6 +119,7 @@ IReferenceSelector * HqlCppTranslator::doBuildRowDeserializeRow(BuildCtx & ctx,
 {
     IHqlExpression * srcRow = expr->queryChild(0);
     IHqlExpression * record = expr->queryRecord();
+    _ATOM serializeForm = expr->queryChild(2)->queryName();
 
     Owned<BoundRow> tempRow = declareLinkedRow(ctx, expr, false);
 
@@ -127,7 +128,7 @@ IReferenceSelector * HqlCppTranslator::doBuildRowDeserializeRow(BuildCtx & ctx,
 
     HqlExprArray args;  
     args.append(*createRowAllocator(ctx, record));
-    args.append(*createRowSerializer(ctx, record, deserializerAtom));
+    args.append(*createSerializer(ctx, record, serializeForm, deserializerAtom));
     args.append(*LINK(srcRow));
     Owned<ITypeInfo> resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));
     OwnedHqlExpr call = bindFunctionCall(rtlDeserializeRowAtom, args, resultType);
@@ -455,14 +456,15 @@ IReferenceSelector * HqlCppTranslator::buildNewRow(BuildCtx & ctx, IHqlExpressio
         }
     case no_getresult:
         {
+            _ATOM serializeForm = diskAtom;  // What if we start using internal in the engines?
             IHqlExpression * seqAttr = expr->queryProperty(sequenceAtom);
             IHqlExpression * nameAttr = expr->queryProperty(namedAtom);
             IHqlExpression * record = expr->queryRecord();
-            OwnedHqlExpr serializedRecord = getSerializedForm(record);
+            OwnedHqlExpr serializedRecord = getSerializedForm(record, serializeForm);
 
             OwnedHqlExpr temp = createDatasetF(no_getresult, LINK(serializedRecord), LINK(seqAttr), LINK(nameAttr), NULL);
             OwnedHqlExpr row = createRow(no_selectnth, LINK(temp), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom)));
-            row.setown(ensureDeserialized(row, expr->queryType()));
+            row.setown(ensureDeserialized(row, expr->queryType(), serializeForm));
             return buildNewRow(ctx, row);
         }
     case no_matchattr:
@@ -1812,50 +1814,101 @@ IHqlExpression * createGetResultFromWorkunitDataset(IHqlExpression * expr)
     return createDataset(no_getresult, LINK(expr->queryRecord()), createComma(LINK(expr->queryProperty(sequenceAtom)), name));
 }
 
-void HqlCppTranslator::buildAssignSerializedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
+void HqlCppTranslator::buildAssignSerializedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, _ATOM serializeForm)
 {
-    OwnedITypeInfo serializedType = getSerializedForm(expr->queryType());
+    OwnedITypeInfo serializedType = getSerializedForm(expr->queryType(), serializeForm);
     assertex(recordTypesMatch(target.queryType(), serializedType));
 
     HqlExprArray args;
-    args.append(*createRowSerializer(ctx, expr->queryRecord(), serializerAtom));
+    args.append(*createSerializer(ctx, expr->queryRecord(), serializeForm, serializerAtom));
     args.append(*LINK(expr));
-    OwnedHqlExpr call = bindFunctionCall(rowset2DatasetXAtom, args);
+
+    _ATOM func;
+    if (target.expr->isDictionary())
+    {
+        assertex(serializeForm == internalAtom);
+        func = rtlSerializeDictionaryAtom;
+    }
+    else if (expr->isDictionary())
+    {
+        assertex(serializeForm == diskAtom);
+        func = rtlSerializeDictionaryToDatasetAtom;
+    }
+    else
+    {
+        if (isGrouped(expr))
+            func = groupedRowset2DatasetXAtom;
+        else
+            func = rowset2DatasetXAtom;
+    }
+
+
+    OwnedHqlExpr call = bindFunctionCall(func, args);
     buildExprAssign(ctx, target, call);
 }
 
-void HqlCppTranslator::buildSerializedDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
+void HqlCppTranslator::buildSerializedDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, _ATOM serializeForm)
 {
     CHqlBoundTarget target;
-    OwnedITypeInfo serializedType = getSerializedForm(expr->queryType());
+    OwnedITypeInfo serializedType = getSerializedForm(expr->queryType(), serializeForm);
     createTempFor(ctx, serializedType, target, typemod_none, FormatBlockedDataset);
-    buildAssignSerializedDataset(ctx, target, expr);
+    buildAssignSerializedDataset(ctx, target, expr, serializeForm);
     tgt.setFromTarget(target);
 }
 
 
-void HqlCppTranslator::buildAssignLinkedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
+void HqlCppTranslator::buildAssignDeserializedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, _ATOM serializeForm)
 {
-    OwnedITypeInfo serializedType = getSerializedForm(target.queryType());
+    OwnedITypeInfo serializedType = getSerializedForm(target.queryType(), serializeForm);
     assertex(recordTypesMatch(serializedType, expr->queryType()));
 
+    _ATOM func;
     IHqlExpression * record = ::queryRecord(target.queryType());
     HqlExprArray args;
-    args.append(*createRowSerializer(ctx, record, deserializerAtom));
+    args.append(*createSerializer(ctx, record, serializeForm, deserializerAtom));
+    if (target.expr->isDictionary())
+    {
+        if (serializeForm == internalAtom)
+        {
+            assertex(expr->isDictionary());
+            func = rtlDeserializeDictionaryAtom;
+        }
+        else if (serializeForm == diskAtom)
+        {
+            assertex(expr->isDataset());
+            func = rtlDeserializeDictionaryFromDatasetAtom;
+            StringBuffer lookupHelperName;
+            buildDictionaryHashClass(record, lookupHelperName);
+            args.append(*createQuoted(lookupHelperName.str(), makeBoolType()));
+        }
+        else
+            throwUnexpected();
+    }
+    else
+    {
+        if (isGrouped(expr))
+            func = groupedDataset2RowsetXAtom;
+        else
+            func = dataset2RowsetXAtom;
+    }
+
     args.append(*LINK(expr));
-    OwnedHqlExpr call = bindFunctionCall(isGrouped(expr) ? groupedDataset2RowsetXAtom : dataset2RowsetXAtom, args, target.queryType());
+    OwnedHqlExpr call = bindFunctionCall(func, args, target.queryType());
     buildExprAssign(ctx, target, call);
 }
 
-void HqlCppTranslator::buildLinkedDataset(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * expr, CHqlBoundExpr & tgt)
+void HqlCppTranslator::buildDeserializedDataset(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * expr, CHqlBoundExpr & tgt, _ATOM serializeForm)
 {
     CHqlBoundTarget target;
     createTempFor(ctx, type, target, typemod_none, FormatLinkedDataset);
 
     if (hasLinkedRow(expr->queryType()))
-        buildDatasetAssign(ctx, target, expr);
+    {
+        throwUnexpected();
+        buildDatasetAssign(ctx, target, expr); // ???
+    }
     else
-        buildAssignLinkedDataset(ctx, target, expr);
+        buildAssignDeserializedDataset(ctx, target, expr, serializeForm);
 
     tgt.setFromTarget(target);
 }
@@ -1863,6 +1916,7 @@ void HqlCppTranslator::buildLinkedDataset(BuildCtx & ctx, ITypeInfo * type, IHql
 
 void HqlCppTranslator::ensureDatasetFormat(BuildCtx & ctx, ITypeInfo * type, CHqlBoundExpr & tgt, ExpressionFormat format)
 {
+    const _ATOM serializeForm = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
     switch (format)
     {
     case FormatBlockedDataset:
@@ -1870,7 +1924,8 @@ void HqlCppTranslator::ensureDatasetFormat(BuildCtx & ctx, ITypeInfo * type, CHq
         {
             OwnedHqlExpr deserializedExpr = tgt.getTranslatedExpr();
             LinkedHqlExpr savedCount = tgt.count;
-            buildSerializedDataset(ctx, deserializedExpr, tgt);
+            assertex(!deserializedExpr->isDictionary());
+            buildSerializedDataset(ctx, deserializedExpr, tgt, serializeForm);
             if (savedCount && !isFixedWidthDataset(deserializedExpr))
                 tgt.count.set(savedCount);
             return;
@@ -1880,7 +1935,13 @@ void HqlCppTranslator::ensureDatasetFormat(BuildCtx & ctx, ITypeInfo * type, CHq
         if (!hasLinkCountedModifier(tgt.queryType()))
         {
             OwnedHqlExpr serializedExpr = tgt.getTranslatedExpr();
-            buildLinkedDataset(ctx, type, serializedExpr, tgt);
+            if (recordTypesMatch(type, tgt.queryType()))
+            {
+                //source is an array of rows, or a simple dataset that doesn't need any transformation
+                buildTempExpr(ctx, serializedExpr, tgt, FormatLinkedDataset);
+            }
+            else
+                buildDeserializedDataset(ctx, type, serializedExpr, tgt, serializeForm);
             return;
         }
         break;
@@ -1888,7 +1949,7 @@ void HqlCppTranslator::ensureDatasetFormat(BuildCtx & ctx, ITypeInfo * type, CHq
         if (!isArrayRowset(tgt.queryType()))
         {
             OwnedHqlExpr serializedExpr = tgt.getTranslatedExpr();
-            buildLinkedDataset(ctx, type, serializedExpr, tgt);
+            buildDeserializedDataset(ctx, type, serializedExpr, tgt, serializeForm);
             return;
         }
         break;
@@ -2046,13 +2107,13 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
         if (isDummySerializeDeserialize(expr))
             doBuildDataset(ctx, expr->queryChild(0)->queryChild(0), tgt, format);
         else
-            buildSerializedDataset(ctx, expr->queryChild(0), tgt);
+            buildSerializedDataset(ctx, expr->queryChild(0), tgt, expr->queryChild(1)->queryName());
         return;
     case no_deserialize:
         if (isDummySerializeDeserialize(expr))
             doBuildDataset(ctx, expr->queryChild(0)->queryChild(0), tgt, format);
         else
-            buildLinkedDataset(ctx, expr->queryType(), expr->queryChild(0), tgt);
+            buildDeserializedDataset(ctx, expr->queryType(), expr->queryChild(0), tgt, expr->queryChild(2)->queryName());
         return;
     case no_datasetfromrow:
         {
@@ -2137,7 +2198,8 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             Owned<IHqlCppDatasetBuilder> builder;
 
             IHqlExpression * record = expr->queryRecord();
-            OwnedHqlExpr serializedRecord = getSerializedForm(record);
+            const _ATOM serializeForm = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
+            OwnedHqlExpr serializedRecord = getSerializedForm(record, serializeForm);
             if (format == FormatNatural)
             {
                 if (record != serializedRecord)
@@ -2329,13 +2391,13 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
         if (isDummySerializeDeserialize(expr))
             buildDatasetAssign(ctx, target, expr->queryChild(0)->queryChild(0));
         else
-            buildAssignSerializedDataset(ctx, target, expr->queryChild(0));
+            buildAssignSerializedDataset(ctx, target, expr->queryChild(0), expr->queryChild(1)->queryName());
         return;
     case no_deserialize:
         if (isDummySerializeDeserialize(expr))
             buildDatasetAssign(ctx, target, expr->queryChild(0)->queryChild(0));
         else
-            buildAssignLinkedDataset(ctx, target, expr->queryChild(0));
+            buildAssignDeserializedDataset(ctx, target, expr->queryChild(0), expr->queryChild(2)->queryName());
         return;
     case no_select:
         {
@@ -2413,17 +2475,18 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
             bool sourceOutOfLine = isArrayRowset(exprType);
             if (sourceOutOfLine != targetOutOfLine)
             {
-                OwnedITypeInfo serializedSourceType = getSerializedForm(exprType);
-                OwnedITypeInfo serializedTargetType = getSerializedForm(to);
+                _ATOM serializeFormat = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
+                OwnedITypeInfo serializedSourceType = getSerializedForm(exprType, serializeFormat);
+                OwnedITypeInfo serializedTargetType = getSerializedForm(to, serializeFormat);
                 if (queryUnqualifiedType(serializedSourceType) == queryUnqualifiedType(serializedTargetType))
                 {
                     if (targetOutOfLine)
                     {
-                        buildAssignLinkedDataset(ctx, target, expr);
+                        buildAssignDeserializedDataset(ctx, target, expr, serializeFormat);
                     }
                     else
                     {
-                        buildAssignSerializedDataset(ctx, target, expr);
+                        buildAssignSerializedDataset(ctx, target, expr, serializeFormat);
                     }
                     return;
                 }
@@ -3899,6 +3962,7 @@ void HqlCppTranslator::doBuildRowAssignProjectRow(BuildCtx & ctx, IReferenceSele
 void HqlCppTranslator::doBuildRowAssignSerializeRow(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
 {
     IHqlExpression * srcRow = expr->queryChild(0);
+    _ATOM serializeForm = expr->queryChild(1)->queryName();
 
     Owned<IReferenceSelector> source = buildNewRow(ctx, srcRow);
 
@@ -3913,7 +3977,7 @@ void HqlCppTranslator::doBuildRowAssignSerializeRow(BuildCtx & ctx, IReferenceSe
     assertex(selfCursor->queryBuilder());
     {
         HqlExprArray args;
-        args.append(*createRowSerializer(ctx, unserializedRecord, serializerAtom));
+        args.append(*createSerializer(ctx, unserializedRecord, serializeForm, serializerAtom));
         args.append(*LINK(srcRow));
 
         Owned<ITypeInfo> type = makeTransformType(expr->queryRecord()->getType());
@@ -4002,7 +4066,7 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * targe
 
     //if record structures are identical, then we must just be able to block copy the information across.
     bool useMemcpy = (sourceRecord == targetRecord) && source->isBinary() && !source->isConditional() && 
-                     !recordRequiresSerialization(sourceRecord);
+                     !recordRequiresLinkCount(sourceRecord);
 
     if (useMemcpy)
     {
@@ -4588,7 +4652,7 @@ void HqlCppTranslator::doBuildExprGetGraphResult(BuildCtx & ctx, IHqlExpression
         }
     }
 
-    bool useLinkCounted = recordRequiresSerialization(expr->queryRecord()) || options.tempDatasetsUseLinkedRows;
+    bool useLinkCounted = recordRequiresLinkCount(expr->queryRecord()) || options.tempDatasetsUseLinkedRows;
 
     OwnedHqlExpr call = buildGetLocalResult(ctx, expr, useLinkCounted);
     switch (expr->queryType()->getTypeCode())

ecl/hqlcpp/hqlcppsys.ecl

@@ -18,9 +18,15 @@
 #define HQLCPPSYS_HPP
 
 const char * cppSystemText[]  = {
-
-    //NB: entrypoint needs to be specified because the identifiers aren't case sensitive.
+    //Definition of all the system functions that are explicitly called by the code generator
+    //Should consider moving this to a reserved module within the ecllibrary.
     "shared dummyRecord := { string1 x };",
+
+    //typedefs to clarify the prototypes.  One day these will become real classes.
+    "shared IOutputRowSerializer := boolean;",
+    "shared IOutputRowDeserializer := boolean;",
+    "shared IHThorHashLookupInfo := boolean;",
+    
     "export InternalCppService := SERVICE",
     //  searchTableStringN(unsigned4 num, string table, string search) : library='eclrtl';
     //  memcpy(void * target, void * src, unsigned len);
@@ -399,6 +405,7 @@ const char * cppSystemText[]  = {
     "   deserializeRaw(data field, boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeRaw';",
     "   data deserializeDataX(boolean o) :  eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeDataX';",
     "   dataset deserializeDatasetX(boolean o) :    eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeDataX';",
+    "   _linkcounted_ dictionary deserializeDictionaryX(boolean _deserializer, boolean _input) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeDictionaryX';",
     "   grouped dataset deserializeGroupedDatasetX(boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeDataX';",
     "   _linkcounted_ dataset deserializeRowsetX(boolean _deserializer, boolean _input) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeRowsetX';",
     "   _linkcounted_ grouped dataset deserializeGroupedRowsetX(boolean _deserializer, boolean _input) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeGroupedRowsetX';",
@@ -422,6 +429,7 @@ const char * cppSystemText[]  = {
     "   serializeRaw(const data field, boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='serializeRaw';",
     "   serializeDataX(const data value, boolean o) :   eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='serializeDataX';",
     "   serializeDatasetX(dataset value, boolean o) :   eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='serializeDataX';",
+    "   serializeDictionaryX(linkcounted dictionary value, boolean _ser, boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='serializeDictionaryX';",
     "   serializeGroupedDatasetX(grouped dataset value, boolean o) :    eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='serializeDataX';",
     "   serializeRowsetX(_array_ dataset value, boolean _ser, boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='serializeRowsetX';",
     "   serializeGroupedRowsetX(_array_ grouped dataset value, boolean _ser, boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='serializeGroupedRowsetX';",
@@ -766,8 +774,8 @@ const char * cppSystemText[]  = {
     "   walkIndirectMetaMember(row _x, boolean _visitor) : omethod,entrypoint='walkIndirectMembers';",
     "   _linkcounted_ dataset linkdataset2linkdataset(_linkcounted_ dataset _x) : include,allocator(false),context,entrypoint='linkdataset2linkdataset';",
 
-    "   unsigned4 rtlSerializeRow(unsigned4 _outLen, dummyRecord _out, boolean _serializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlSerializeRow';",
     "   dummyRecord rtlSerializeToBuilder(boolean _serializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlSerializeToBuilder';",
+    "   dummyRecord rtlDeserializeToBuilder(boolean _serializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlDeserializeToBuilder';",
     "   _linkcounted_ row(dummyRecord) rtlDeserializeRow(boolean _allocator, boolean _deserializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlDeserializeRow';",
 
     "   releaseRow(row _x) : include,entrypoint='rtlReleaseRow';",
@@ -775,26 +783,37 @@ const char * cppSystemText[]  = {
     "   linkRow(row _x) : include,entrypoint='rtlLinkRow';",
     "   _linkcounted_ dataset linkRowset(_linkcounted_ dataset _x) : include,allocator(false),entrypoint='rtlLinkRowset';",
 
-    "   _linkcounted_ dataset deserializerRowsetHelper(boolean _deserializer, boolean _input) : eclrtl,include,entrypoint='rtlDeserializeRowset';",
-    "   serializerRowsetHelper(boolean _output, boolean _serializer, _linkcounted_ dataset _x) : eclrtl,include,entrypoint='rtlSerializeRowset';",
+    "   _linkcounted_ dataset deserializeChildRowsetFromStream(boolean _deserializer, boolean _input) : eclrtl,include,entrypoint='rtlDeserializeChildRowset';",
+    "   serializeChildRowsetToStream(boolean _output, boolean _serializer, _linkcounted_ dataset _x) : eclrtl,include,entrypoint='rtlSerializeChildRowset';",
+
+    "   linkcounted dictionary deserializeChildDictionaryFromStream(boolean _deserializer, boolean _input) : eclrtl,include,entrypoint='rtlDeserializeChildDictionary';",
+    "   linkcounted dictionary deserializeChildDictionaryFromDatasetFromStream(boolean _deserializer, boolean _hasher, boolean _input) : eclrtl,include,entrypoint='rtlDeserializeChildDictionaryFromDataset';",
+    "   serializeChildDictionaryToStream(boolean _output, boolean _serializer, _linkcounted_ dictionary _x) : eclrtl,include,entrypoint='rtlSerializeChildDictionary';",
+    "   serializeChildDictionaryToDatasetToStream(boolean _output, boolean _serializer, _linkcounted_ dictionary _x) : eclrtl,include,entrypoint='rtlSerializeChildDictionaryToDataset';",
 
+    "   dictionary rtlSerializeDictionary(IOutputRowSerializer _ser, linkcounted dictionary x) : eclrtl,include,library='eclrtl',entrypoint='rtlSerializeDictionary';",
+    "   dataset rtlSerializeDictionaryToDataset(IOutputRowSerializer _ser, linkcounted dictionary x) : eclrtl,include,library='eclrtl',entrypoint='rtlSerializeDictionaryToDataset';",
+    "   linkcounted dictionary rtlDeserializeDictionary(IOutputRowDeserializer _ser, dictionary x) : eclrtl,include,library='eclrtl',entrypoint='rtlDeserializeDictionary';",
+    "   linkcounted dictionary rtlDeserializeDictionaryFromDataset(IOutputRowDeserializer _ser, IHThorHashLookupInfo _hashInfo, dataset x) : eclrtl,include,library='eclrtl',entrypoint='rtlDeserializeDictionaryFromDataset';",
+
+
+//Methods of IRowDeserializerSource
     "   row(dummyRecord) deserializerPeek(unsigned4 _maxSize) : omethod,entrypoint='peek';",
     "   unsigned4 deserializerBeginNested() : omethod,entrypoint='beginNested';",
     "   boolean deserializerFinishedNested(unsigned4 pos) : omethod,entrypoint='finishedNested';",
-
     "   unsigned4 deserializerReadN(data _target) : omethod,entrypoint='read';",
     "   unsigned4 deserializerReadSize() : omethod,entrypoint='readSize';",
     "   unsigned4 deserializerReadPackedInt(data _target) : omethod,entrypoint='readPackedInt';",
     "   unsigned4 deserializerReadUtf8(boolean rowBuilder, unsigned4 offset, unsigned4 fixedSize, unsigned4 len) : omethod,entrypoint='readUtf8';",
     "   unsigned4 deserializerReadVStr(boolean rowBuilder, unsigned4 offset, unsigned4 fixedSize) : omethod,entrypoint='readVStr';",
     "   unsigned4 deserializerReadVUni(boolean rowBuilder, unsigned4 offset, unsigned4 fixedSize) : omethod,entrypoint='readVUni';",
-
     "   deserializerSkipN(unsigned4 _size) : omethod,entrypoint='skip';",
     "   deserializerSkipPacketInt() : omethod,entrypoint='skipPackedInt';",
     "   deserializerSkipUtf8(unsigned4 _size) : omethod,entrypoint='skipUtf8';",
     "   deserializerSkipVStr() : omethod,entrypoint='skipVStr';",
     "   deserializerSkipVUni() : omethod,entrypoint='skipVUni';",
 
+//Methods of IRowSerializerTarget
     "   serializerPut(const data _target) : omethod,entrypoint='put';",
     "   unsigned4 serializerBeginNested() : omethod,entrypoint='beginNested';",
     "   serializerEndNested(unsigned4 pos) : omethod,entrypoint='endNested';",

ecl/hqlcpp/hqlcset.cpp

@@ -721,8 +721,7 @@ BoundRow * InlineLinkedDictionaryCursor::buildSelectMap(BuildCtx & ctx, IHqlExpr
     IHqlExpression *record = ds->queryRecord();
 
     StringBuffer lookupHelperName;
-    OwnedHqlExpr dict = createDictionary(no_null, LINK(record));
-    translator.buildDictionaryHashClass(record, dict, lookupHelperName);
+    translator.buildDictionaryHashClass(record, lookupHelperName);
     CHqlBoundTarget target;
     target.expr.set(tempRow->queryBound());
 
@@ -743,8 +742,7 @@ void InlineLinkedDictionaryCursor::buildInDataset(BuildCtx & ctx, IHqlExpression
     IHqlExpression *record = ds->queryRecord();
 
     StringBuffer lookupHelperName;
-    OwnedHqlExpr dict = createDictionary(no_null, LINK(record));
-    translator.buildDictionaryHashClass(record, dict, lookupHelperName);
+    translator.buildDictionaryHashClass(record, lookupHelperName);
 
     HqlExprArray args;
     args.append(*createQuoted(lookupHelperName, makeBoolType()));
@@ -1416,10 +1414,10 @@ BoundRow * DatasetBuilderBase::buildCreateRow(BuildCtx & ctx)
     return translator.bindSelf(ctx, dataset, builderName);
 }
 
-BoundRow * DatasetBuilderBase::buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput)
+BoundRow * DatasetBuilderBase::buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput, _ATOM serializeForm)
 {
     StringBuffer serializerInstanceName;
-    translator.ensureRowSerializer(serializerInstanceName, ctx, record, deserializerAtom);
+    translator.ensureRowSerializer(serializerInstanceName, ctx, record, serializeForm, deserializerAtom);
 
     StringBuffer s;
     s.append(instanceName).append(".deserializeRow(*");
@@ -1797,8 +1795,7 @@ void LinkedDictionaryBuilder::buildDeclare(BuildCtx & ctx)
     translator.ensureRowAllocator(allocatorName, ctx, record, curActivityId);
 
     StringBuffer lookupHelperName;
-    OwnedHqlExpr dict = createDictionary(no_null, record.getLink()); // MORE - is the actual dict not available?
-    translator.buildDictionaryHashClass(record, dict, lookupHelperName);
+    translator.buildDictionaryHashClass(record, lookupHelperName);
 
     decl.append("RtlLinkedDictionaryBuilder ").append(instanceName).append("(");
     decl.append(allocatorName).append(", &").append(lookupHelperName);

ecl/hqlcpp/hqlcset.ipp

@@ -229,7 +229,7 @@ public:
     IMPLEMENT_IINTERFACE
 
     virtual BoundRow * buildCreateRow(BuildCtx & ctx);
-    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput);
+    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput, _ATOM serializeForm);
     virtual void finishRow(BuildCtx & ctx, BoundRow * selfCursor);
 
 protected:
@@ -266,7 +266,7 @@ public:
 
     virtual void buildDeclare(BuildCtx & ctx);
     virtual BoundRow * buildCreateRow(BuildCtx & ctx);
-    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput) { throwUnexpected(); }
+    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput, _ATOM serializeForm) { throwUnexpected(); }
     virtual void buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target);
     virtual void buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound);
     virtual void finishRow(BuildCtx & ctx, BoundRow * selfCursor);
@@ -355,7 +355,7 @@ public:
 
     virtual void buildDeclare(BuildCtx & ctx);
     virtual BoundRow * buildCreateRow(BuildCtx & ctx);
-    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput) { throwUnexpected(); }
+    virtual BoundRow * buildDeserializeRow(BuildCtx & ctx, IHqlExpression * serializedInput, _ATOM serializeForm) { throwUnexpected(); }
     virtual void buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target);
     virtual void buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound);
     virtual void finishRow(BuildCtx & ctx, BoundRow * selfCursor);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1509,7 +1509,7 @@ void HqlCppTranslator::queryAddResultDependancy(ABoundActivity & whoAmIActivity,
 
 bool HqlCppTranslator::tempRowRequiresFinalize(IHqlExpression * record) const
 {
-    if (recordRequiresSerialization(record) || options.finalizeAllRows)
+    if (recordRequiresDestructor(record) || options.finalizeAllRows)
         return true;
     if (options.finalizeAllVariableRows && isVariableSizeRecord(record))
         return true;
@@ -2732,15 +2732,15 @@ void DatasetSelector::modifyOp(BuildCtx & ctx, IHqlExpression * expr, node_opera
 }
 
 
-void DatasetSelector::buildDeserialize(BuildCtx & ctx, IHqlExpression * helper)
+void DatasetSelector::buildDeserialize(BuildCtx & ctx, IHqlExpression * helper, _ATOM serializeForm)
 {
-    column->buildDeserialize(translator, ctx, this, helper);
+    column->buildDeserialize(translator, ctx, this, helper, serializeForm);
 }
 
 
-void DatasetSelector::buildSerialize(BuildCtx & ctx, IHqlExpression * helper)
+void DatasetSelector::buildSerialize(BuildCtx & ctx, IHqlExpression * helper, _ATOM serializeForm)
 {
-    column->buildSerialize(translator, ctx, this, helper);
+    column->buildSerialize(translator, ctx, this, helper, serializeForm);
 }
 
 /* In selector: not linked. Return: linked */
@@ -3402,7 +3402,7 @@ IHqlExpression * HqlCppTranslator::createRowAllocator(BuildCtx & ctx, IHqlExpres
 }
 
 
-void HqlCppTranslator::buildMetaSerializerClass(BuildCtx & ctx, IHqlExpression * record, const char * serializerName)
+void HqlCppTranslator::buildMetaSerializerClass(BuildCtx & ctx, IHqlExpression * record, const char * serializerName, _ATOM serializeForm)
 {
     StringBuffer s;
     GlobalClassBuilder serializer(*this, ctx, serializerName, "COutputRowSerializer", "IOutputRowSerializer");
@@ -3426,10 +3426,10 @@ void HqlCppTranslator::buildMetaSerializerClass(BuildCtx & ctx, IHqlExpression *
         OwnedHqlExpr size = getRecordSize(row->querySelector());
         CHqlBoundExpr boundSize;
         buildExpr(serializectx, size, boundSize);
-        if (recordRequiresSerialization(record))
+        if (recordRequiresSerialization(record, serializeForm))
         {
             Owned<IReferenceSelector> selector = buildActiveRow(serializectx, row->querySelector());
-            selector->buildSerialize(serializectx, helper);
+            selector->buildSerialize(serializectx, helper, serializeForm);
         }
         else
         {
@@ -3446,7 +3446,7 @@ void HqlCppTranslator::buildMetaSerializerClass(BuildCtx & ctx, IHqlExpression *
     serializer.completeClass(RowMetaPrio);
 }
 
-void HqlCppTranslator::buildMetaDeserializerClass(BuildCtx & ctx, IHqlExpression * record, const char * deserializerName)
+void HqlCppTranslator::buildMetaDeserializerClass(BuildCtx & ctx, IHqlExpression * record, const char * deserializerName, _ATOM serializeForm)
 {
     StringBuffer s;
     GlobalClassBuilder deserializer(*this, ctx, deserializerName, "COutputRowDeserializer", "IOutputRowDeserializer");
@@ -3470,7 +3470,7 @@ void HqlCppTranslator::buildMetaDeserializerClass(BuildCtx & ctx, IHqlExpression
 
         OwnedHqlExpr helper = createVariable("in", makeBoolType());
         Owned<IReferenceSelector> selector = buildActiveRow(deserializectx, row->querySelector());
-        selector->buildDeserialize(deserializectx, helper);
+        selector->buildDeserialize(deserializectx, helper, serializeForm);
         buildReturnRecordSize(deserializectx, row);
     }
 
@@ -4014,7 +4014,7 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
     contextAvailable = false;
     metas.append(*search.getLink());
     StringBuffer s;
-    StringBuffer serializerName, deserializerName, prefetcherName;
+    StringBuffer serializerName, deserializerName, prefetcherName, internalSerializerName, internalDeserializerName;
 
     StringBuffer endText;
 
@@ -4034,8 +4034,15 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
     {
         if (recordRequiresDestructor(record))
             flags |= MDFneeddestruct;
-        if (recordRequiresSerialization(record))
-            flags |= MDFneedserialize;
+        if (recordRequiresSerialization(record, diskAtom))
+            flags |= MDFneedserializedisk;
+        if (recordRequiresSerialization(record, internalAtom))
+            flags |= MDFneedserializeinternal;
+
+        const unsigned serializeFlags = (flags & MDFneedserializemask);
+        if ((serializeFlags == MDFneedserializemask) && !recordSerializationDiffers(record, diskAtom, internalAtom))
+            flags |= MDFdiskmatchesinternal;
+
         if (maxRecordSizeUsesDefault(record))
             flags |= MDFunknownmaxlength;
         useTypeForXML = true;
@@ -4055,21 +4062,36 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
     {
         //Serialization classes need to be generated for all meta information - because they may be called by parent row classes
         //however, the CFixedOutputMetaData base class contains a default implementation - reducing the required code.
-        if (map && (!map->isFixedWidth() || (flags & MDFneedserialize)))
+        if (map && (!map->isFixedWidth() || (flags & MDFneedserializemask)))
         {
             //Base class provides a default variable width implementation
-            if (flags & MDFneedserialize)
+            if (flags & MDFneedserializedisk)
             {
                 serializerName.append("s").append(instance.metaName);
-                buildMetaSerializerClass(declarectx, record, serializerName.str());
+                buildMetaSerializerClass(declarectx, record, serializerName.str(), diskAtom);
+            }
+            bool needInternalSerializer = ((flags & MDFneedserializeinternal) && recordSerializationDiffers(record, diskAtom, internalAtom));
+
+            if (needInternalSerializer)
+            {
+                internalSerializerName.append("si").append(instance.metaName);
+                buildMetaSerializerClass(declarectx, record, internalSerializerName.str(), internalAtom);
             }
+
+            //MORE:
             //still generate a deserialize for the variable width case because it offers protection
             //against accessing out of bounds data
             deserializerName.append("d").append(instance.metaName);
-            buildMetaDeserializerClass(declarectx, record, deserializerName.str());
+            buildMetaDeserializerClass(declarectx, record, deserializerName.str(), diskAtom);
+
+            if (needInternalSerializer)
+            {
+                internalDeserializerName.append("di").append(instance.metaName);
+                buildMetaDeserializerClass(declarectx, record, internalDeserializerName.str(), internalAtom);
+            }
 
             //The base class implements prefetch using the serialized meta so no need to generate...
-            if (!(flags & MDFneedserialize))
+            if (!(flags & MDFneedserializemask))
             {
                 prefetcherName.append("p").append(instance.metaName);
                 if (!buildMetaPrefetcherClass(declarectx, record, prefetcherName))
@@ -4143,19 +4165,19 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
             }
 
             if (record)
-                generateMetaRecordSerialize(metactx, record, serializerName.str(), deserializerName.str(), prefetcherName.str());
+                generateMetaRecordSerialize(metactx, record, serializerName.str(), deserializerName.str(), internalSerializerName.str(), internalDeserializerName.str(), prefetcherName.str());
 
             if (flags != (MDFhasserialize|MDFhasxml))
                 doBuildUnsignedFunction(metactx, "getMetaFlags", flags);
 
-            if (flags & MDFneedserialize)
+            if (flags & MDFneedserializedisk)
             {
-                OwnedHqlExpr serializedRecord = getSerializedForm(record);
+                OwnedHqlExpr serializedRecord = getSerializedForm(record, diskAtom);
 
                 MetaInstance serializedMeta(*this, serializedRecord, false);
                 buildMetaInfo(serializedMeta);
                 StringBuffer s;
-                s.append("virtual IOutputMetaData * querySerializedMeta() { return &").append(serializedMeta.queryInstanceObject()).append("; }");
+                s.append("virtual IOutputMetaData * querySerializedDiskMeta() { return &").append(serializedMeta.queryInstanceObject()).append("; }");
                 metactx.addQuoted(s);
             }
         }
@@ -4325,7 +4347,7 @@ protected:
 
 
 
-void HqlCppTranslator::generateMetaRecordSerialize(BuildCtx & ctx, IHqlExpression * record, const char * serializerName, const char * deserializerName, const char * prefetcherName)
+void HqlCppTranslator::generateMetaRecordSerialize(BuildCtx & ctx, IHqlExpression * record, const char * diskSerializerName, const char * diskDeserializerName, const char * internalSerializerName, const char * internalDeserializerName, const char * prefetcherName)
 {
     OwnedHqlExpr dataset = createDataset(no_null, LINK(record));
 
@@ -4348,30 +4370,50 @@ void HqlCppTranslator::generateMetaRecordSerialize(BuildCtx & ctx, IHqlExpressio
         builder.walkRecord(walkctx, dataset, record);
     }
 
-    if (serializerName && *serializerName)
+    if (diskSerializerName && *diskSerializerName)
     {
         BuildCtx serializectx(ctx);
-        serializectx.addQuotedCompound("virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)");
+        serializectx.addQuotedCompound("virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)");
 
         StringBuffer s;
-        s.append("return cr").append(serializerName).append("(ctx, activityId);");
+        s.append("return cr").append(diskSerializerName).append("(ctx, activityId);");
         serializectx.addQuoted(s);
     }
 
-    if (deserializerName && *deserializerName)
+    if (diskDeserializerName && *diskDeserializerName)
     {
         BuildCtx deserializectx(ctx);
-        deserializectx.addQuotedCompound("virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId)");
+        deserializectx.addQuotedCompound("virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId)");
 
         StringBuffer s;
-        s.append("return cr").append(deserializerName).append("(ctx, activityId);");
+        s.append("return cr").append(diskDeserializerName).append("(ctx, activityId);");
+        deserializectx.addQuoted(s);
+    }
+
+    if (internalSerializerName && *internalSerializerName)
+    {
+        BuildCtx serializectx(ctx);
+        serializectx.addQuotedCompound("virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId)");
+
+        StringBuffer s;
+        s.append("return cr").append(internalSerializerName).append("(ctx, activityId);");
+        serializectx.addQuoted(s);
+    }
+
+    if (internalDeserializerName && *internalDeserializerName)
+    {
+        BuildCtx deserializectx(ctx);
+        deserializectx.addQuotedCompound("virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId)");
+
+        StringBuffer s;
+        s.append("return cr").append(internalDeserializerName).append("(ctx, activityId);");
         deserializectx.addQuoted(s);
     }
 
     if (prefetcherName && *prefetcherName)
     {
         BuildCtx deserializectx(ctx);
-        deserializectx.addQuotedCompound("virtual CSourceRowPrefetcher * createRawRowPrefetcher(unsigned activityId)");
+        deserializectx.addQuotedCompound("virtual CSourceRowPrefetcher * doCreateDiskPrefetcher(unsigned activityId)");
 
         StringBuffer s;
         s.append("return new ").append(prefetcherName).append("(activityId);");
@@ -4417,7 +4459,7 @@ void HqlCppTranslator::buildMetaForSerializedRecord(StringBuffer & name, IHqlExp
         buildMetaForRecord(name, record);
 }
 
-void HqlCppTranslator::ensureRowSerializer(StringBuffer & serializerName, BuildCtx & ctx, IHqlExpression * record, _ATOM kind)
+void HqlCppTranslator::ensureRowSerializer(StringBuffer & serializerName, BuildCtx & ctx, IHqlExpression * record, _ATOM format, _ATOM kind)
 {
     OwnedHqlExpr marker = createAttribute(serializerInstanceMarkerAtom, LINK(record->queryBody()), createAttribute(kind));
     HqlExprAssociation * match = ctx.queryMatchExpr(marker);
@@ -4435,20 +4477,25 @@ void HqlCppTranslator::ensureRowSerializer(StringBuffer & serializerName, BuildC
     getInvariantMemberContext(ctx, &declarectx, &callctx, true, false);
 
     StringBuffer s;
-    if (kind == serializerAtom)
-        s.append("Owned<IOutputRowSerializer> ").append(uid).append(";");
-    else
-        s.append("Owned<IOutputRowDeserializer> ").append(uid).append(";");
+    const char * kindText = (kind == serializerAtom) ? "Serializer" : "Deserializer";
+
+    s.append("Owned<IOutputRow").append(kindText).append("> ").append(uid).append(";");
     declarectx->addQuoted(s);
 
     MetaInstance meta(*this, record, false);
     buildMetaInfo(meta);
 
     s.clear().append(uid).append(".setown(").append(meta.queryInstanceObject());
-    if (kind == serializerAtom)
-        s.append(".createRowSerializer");
+
+    if (format == diskAtom)
+        s.append(".createDisk").append(kindText);
+    else if (format == internalAtom)
+        s.append(".createInternal").append(kindText);
     else
-        s.append(".createRowDeserializer");
+    {
+        UNIMPLEMENTED;
+    }
+
     s.append("(ctx, ");
     OwnedHqlExpr activityId = getCurrentActivityId(ctx);
     generateExprCpp(s, activityId);
@@ -4486,7 +4533,7 @@ void HqlCppTranslator::ensureRowPrefetcher(StringBuffer & prefetcherName, BuildC
     buildMetaInfo(meta);
 
     s.clear().append(uid).append(".setown(").append(meta.queryInstanceObject());
-    s.append(".createRowPrefetcher(ctx, ");
+    s.append(".createDiskPrefetcher(ctx, ");
     OwnedHqlExpr activityId = getCurrentActivityId(ctx);
     generateExprCpp(s, activityId);
     s.append("));");
@@ -4498,10 +4545,10 @@ void HqlCppTranslator::ensureRowPrefetcher(StringBuffer & prefetcherName, BuildC
 }
 
 
-IHqlExpression * HqlCppTranslator::createRowSerializer(BuildCtx & ctx, IHqlExpression * record, _ATOM kind)
+IHqlExpression * HqlCppTranslator::createSerializer(BuildCtx & ctx, IHqlExpression * record, _ATOM form, _ATOM kind)
 {
     StringBuffer serializerName;
-    ensureRowSerializer(serializerName, ctx, record, kind);
+    ensureRowSerializer(serializerName, ctx, record, form, kind);
     return createQuoted(serializerName.str(), makeBoolType());
 }
 
@@ -4703,7 +4750,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
 
                 //NB: The result type will be overridden when this function is bound
                 ensureSerialized = false;
-                args.append(*createRowSerializer(ctx, record, deserializerAtom));
+                args.append(*createSerializer(ctx, record, diskAtom, deserializerAtom));
                 overrideType.setown(setLinkCountedAttr(type, true));
                 func = getResultDictionaryAtom;
             }
@@ -4718,7 +4765,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
                 {
                     ensureSerialized = false;
                     args.append(*createRowAllocator(ctx, record));
-                    args.append(*createRowSerializer(ctx, record, deserializerAtom));
+                    args.append(*createSerializer(ctx, record, internalAtom, deserializerAtom));
                     args.append(*createConstant(isGrouped(expr)));
                     overrideType.setown(setLinkCountedAttr(overrideType, true));
                     func = getResultRowsetAtom;
@@ -4734,7 +4781,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
             }
 
             if (ensureSerialized && record)
-                record.setown(getSerializedForm(record));
+                record.setown(getSerializedForm(record, diskAtom));
 
             if (record && (seqValue == ResultSequenceStored))
             {
@@ -4771,7 +4818,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
             if (ttc == type_dictionary)
             {
                 StringBuffer lookupHelperName;
-                buildDictionaryHashClass(expr->queryRecord(), expr, lookupHelperName);
+                buildDictionaryHashClass(expr->queryRecord(), lookupHelperName);
 
                 lookupHelperName.insert(0, "&");    // MORE: Should this be passed by reference instead - it isn't optional
                 args.append(*createQuoted(lookupHelperName.str(), makeBoolType()));
@@ -4974,7 +5021,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
     case type_row:
         {
             CHqlBoundExpr boundLength;
-            OwnedHqlExpr serialized = ::ensureSerialized(value);
+            OwnedHqlExpr serialized = ::ensureSerialized(value, diskAtom);
             func = setResultRawAtom;
             Owned<IReferenceSelector> ds = buildNewRow(ctx, serialized);
             OwnedHqlExpr size = createSizeof(ds->queryExpr());
@@ -5321,7 +5368,7 @@ void HqlCppTranslator::buildHashOfExprsClass(BuildCtx & ctx, const char * name,
 }
 
 
-void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &funcName)
+void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, StringBuffer &funcName)
 {
     BuildCtx declarectx(*code, declareAtom);
     OwnedHqlExpr attr = createAttribute(lookupAtom, LINK(record));
@@ -5334,8 +5381,13 @@ void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, IHqlExpr
         appendUniqueId(lookupHelperName.append("lu"), getConsistentUID(record));
 
         BuildCtx classctx(declarectx);
+        //I suspect all the priorities should be killed.  This is here because you can have meta functions accessing the
+        //dictionary hash functions.
+        classctx.setNextPriority(RowMetaPrio);
+
         beginNestedClass(classctx, lookupHelperName, "IHThorHashLookupInfo");
         HqlExprArray keyedFields;
+        OwnedHqlExpr dictionary = createDataset(no_null, LINK(record));
         IHqlExpression * payload = record->queryProperty(_payload_Atom);
         unsigned payloadSize = payload ? (unsigned)getIntValue(payload->queryChild(0)) : 0;
         unsigned max = record->numChildren() - payloadSize;
@@ -5366,7 +5418,7 @@ void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, IHqlExpr
 void HqlCppTranslator::buildDictionaryHashMember(BuildCtx & ctx, IHqlExpression *dictionary, const char * memberName)
 {
     StringBuffer lookupHelperName;
-    buildDictionaryHashClass(dictionary->queryRecord(), dictionary, lookupHelperName);
+    buildDictionaryHashClass(dictionary->queryRecord(), lookupHelperName);
 
     BuildCtx funcctx(ctx);
     StringBuffer s;
@@ -6616,7 +6668,7 @@ void HqlCppTranslator::buildRootActivity(BuildCtx & ctx, IHqlExpression * expr)
 
 void HqlCppTranslator::buildRecordSerializeExtract(BuildCtx & ctx, IHqlExpression * memoryRecord)
 {
-    OwnedHqlExpr serializedRecord = getSerializedForm(memoryRecord);
+    OwnedHqlExpr serializedRecord = getSerializedForm(memoryRecord, diskAtom);
     OwnedHqlExpr serializedDataset = createDataset(no_null, LINK(serializedRecord));
     OwnedHqlExpr memoryDataset = createDataset(no_anon, LINK(memoryRecord));
 
@@ -6647,7 +6699,7 @@ void HqlCppTranslator::buildRecordSerializeExtract(BuildCtx & ctx, IHqlExpressio
         BoundRow * left = bindTableCursor(ctx, memoryDataset, "left");
         OwnedHqlExpr rhs = ensureActiveRow(left->querySelector());
 
-        OwnedHqlExpr serializedRow = ::ensureSerialized(rhs);
+        OwnedHqlExpr serializedRow = ::ensureSerialized(rhs, diskAtom); // Ensure this is always accessed as disk serialized
 
         buildAssign(ctx, self->querySelector(), serializedRow);
         buildReturnRecordSize(ctx, self);
@@ -7075,7 +7127,7 @@ void HqlCppTranslator::doBuildSerialize(BuildCtx & ctx, _ATOM name, IHqlExpressi
     callProcedure(ctx, name, args);
 }
 
-void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildCtx & serializectx, BuildCtx & deserializectx, const char * inBufferName, const char * outBufferName)
+void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildCtx & serializectx, BuildCtx & deserializectx, const char * inBufferName, const char * outBufferName, _ATOM serializeForm)
 {
     CHqlBoundExpr value;
     value.setFromTarget(variable);
@@ -7133,10 +7185,15 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
                 IHqlExpression * record = ::queryRecord(type);
                 if (hasLinkCountedModifier(type))
                 {
-                    deserializeArgs.append(*createRowSerializer(deserializectx, record, deserializerAtom));
+                    deserializeArgs.append(*createSerializer(deserializectx, record, serializeForm, deserializerAtom));
 
-                    serializeArgs.append(*createRowSerializer(serializectx, record, serializerAtom));
-                    if (tc == type_groupedtable)
+                    serializeArgs.append(*createSerializer(serializectx, record, serializeForm, serializerAtom));
+                    if (tc == type_dictionary)
+                    {
+                        serializeName = serializeDictionaryXAtom;
+                        deserializeName = deserializeDictionaryXAtom;
+                    }
+                    else if (tc == type_groupedtable)
                     {
                         serializeName = serializeGroupedRowsetXAtom;
                         deserializeName = deserializeGroupedRowsetXAtom;
@@ -7149,8 +7206,12 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
                 }
                 else
                 {
-                    assertex(!recordRequiresSerialization(record));
-                    if (tc == type_groupedtable)
+                    assertex(!recordRequiresSerialization(record, serializeForm));
+                    if (tc == type_dictionary)
+                    {
+                        throwUnexpected();
+                    }
+                    else if (tc == type_groupedtable)
                     {
                         serializeName = serializeGroupedDatasetXAtom;
                         deserializeName = deserializeGroupedDatasetXAtom;
@@ -7169,13 +7230,13 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
                 IHqlExpression * record = ::queryRecord(type);
                 assertex(hasWrapperModifier(type));
 
-                serializeArgs.append(*createRowSerializer(serializectx, record, serializerAtom));
+                serializeArgs.append(*createSerializer(serializectx, record, serializeForm, serializerAtom));
                 serializeArgs.append(*createVariable(outBufferName, makeBoolType()));
                 buildFunctionCall(serializectx, serializeRowAtom, serializeArgs);
 
 
                 deserializeArgs.append(*createRowAllocator(deserializectx, record));
-                deserializeArgs.append(*createRowSerializer(deserializectx, record, deserializerAtom));
+                deserializeArgs.append(*createSerializer(deserializectx, record, serializeForm, deserializerAtom));
                 deserializeArgs.append(*createVariable(inBufferName, makeBoolType()));
                 Owned<ITypeInfo> resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));
                 OwnedHqlExpr call = bindFunctionCall(deserializeRowAtom, deserializeArgs, resultType);
@@ -7572,7 +7633,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             case type_record:
             case type_row:
                 {
-                    OwnedHqlExpr record = getSerializedForm(child->queryRecord());
+                    OwnedHqlExpr record = getSerializedForm(child->queryRecord(), diskAtom);
                     ColumnToOffsetMap * map = queryRecordOffsetMap(record);
                     if (map->isFixedWidth())
                         size = map->getFixedRecordSize();
@@ -7610,7 +7671,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             case type_record:
             case type_row:
                 {
-                    OwnedHqlExpr record = getSerializedForm(child->queryRecord());
+                    OwnedHqlExpr record = getSerializedForm(child->queryRecord(), diskAtom);
                     size = getMinRecordSize(record);
                 }
                 break;
@@ -7690,7 +7751,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             case type_row:
                 {
                     e->Release();
-                    OwnedHqlExpr record = getSerializedForm(child->queryRecord());
+                    OwnedHqlExpr record = getSerializedForm(child->queryRecord(), diskAtom);
                     ColumnToOffsetMap * map = queryRecordOffsetMap(record);
                     if (map->isFixedWidth())
                     {
@@ -9523,7 +9584,7 @@ void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * datase
     StringBuffer s;
 
     //Ensure the ECL for the record reflects its serialized form, not the internal form
-    OwnedHqlExpr record = getSerializedForm(dataset->queryRecord());
+    OwnedHqlExpr record = getSerializedForm(dataset->queryRecord(), diskAtom);
     appendUniqueId(eclFuncName.append("ecl"), getConsistentUID(record));
 
     BuildCtx declarectx(*code, declareAtom);
@@ -9559,7 +9620,7 @@ void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * datase
 void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta)
 {
     IHqlExpression * payload = expr ? expr->queryProperty(_payload_Atom) : NULL;
-    OwnedHqlExpr exprToCrc = getSerializedForm(dataset->queryRecord());
+    OwnedHqlExpr exprToCrc = getSerializedForm(dataset->queryRecord(), diskAtom);
     unsigned payloadSize = 1;
     if (payload)
         payloadSize = (unsigned)getIntValue(payload->queryChild(0)) + payloadDelta;
@@ -9779,7 +9840,7 @@ IDefRecordElement * HqlCppTranslator::createMetaRecord(IHqlExpression * record)
 
 IHqlExpression * HqlCppTranslator::getSerializedLayoutFunction(IHqlExpression * record, unsigned numKeyedFields)
 {
-    OwnedHqlExpr serializedRecord = getSerializedForm(record);
+    OwnedHqlExpr serializedRecord = getSerializedForm(record, diskAtom);
     OwnedHqlExpr search = createAttribute(indexLayoutMarkerAtom, LINK(serializedRecord), getSizetConstant(numKeyedFields));
 
     BuildCtx declarectx(*code, declareAtom);
@@ -10345,7 +10406,7 @@ void HqlCppTranslator::doAddSchemaFields(IHqlExpression * record, MemoryBuffer &
 
 void HqlCppTranslator::getRecordECL(IHqlExpression * deserializedRecord, StringBuffer & eclText)
 {
-    OwnedHqlExpr record = getSerializedForm(deserializedRecord);
+    OwnedHqlExpr record = getSerializedForm(deserializedRecord, diskAtom);
     if ((options.maxRecordSize != MAX_RECORD_SIZE) && maxRecordSizeUsesDefault(record))
     {
         //Add an explicit record size if default max record size
@@ -10417,7 +10478,7 @@ IWUResult * HqlCppTranslator::createDatasetResultSchema(IHqlExpression * sequenc
     result->setResultSchemaRaw(schema.length(), schema.toByteArray());
     result->setResultScalar(false);
 
-    OwnedHqlExpr serialRecord = getSerializedForm(record);
+    OwnedHqlExpr serialRecord = getSerializedForm(record, diskAtom);
     OwnedHqlExpr ds = createDataset(no_anon, LINK(serialRecord));
     MetaInstance meta(*this, serialRecord, false);
     buildMetaInfo(meta);
@@ -10429,7 +10490,7 @@ IWUResult * HqlCppTranslator::createDatasetResultSchema(IHqlExpression * sequenc
     if (createTransformer)
     {
         OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(serialRecord);
-        Owned<IHqlExpression> transformedRecord = getSimplifiedRecord(noVirtualRecord, false);
+        Owned<IHqlExpression> transformedRecord = getFileViewerRecord(noVirtualRecord, false);
         if (transformedRecord)
         {
             OwnedHqlExpr ds = createDataset(no_anon, LINK(noVirtualRecord));
@@ -11131,7 +11192,7 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
     }
 }
 
-void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear)
+void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear, node_operator serializeOp, _ATOM serialForm)
 {
     ForEachChild(i, record)
     {
@@ -11146,13 +11207,22 @@ void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression *
                 {
                     Owned<IHqlExpression> self = tgtDataset.mapScalar(&tgtSelects.item(matchIndex), selfSelect);
                     Owned<IHqlExpression> left = srcDataset.mapScalar(&srcSelects.item(matchIndex), leftSelect);
+                    if (self->queryType() != left->queryType())
+                    {
+                        HqlExprArray args;
+                        args.append(*LINK(left));
+                        if (serializeOp == no_deserialize)
+                            args.append(*LINK(self->queryRecord()));
+                        args.append(*createAttribute(serialForm));
+                        left.setown(createWrapper(serializeOp, self->queryType(), args));
+                    }
                     buildAssign(ctx, self, left);
                     //Note, we could stop here if needToClear and all fields have been assigned, and all the following fields are fixed width.
                     // but not really sure it is worth it.
                 }
                 else if (cur->isDatarow())
                 {
-                    generateSerializeAssigns(ctx, cur->queryRecord(), selected, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
+                    generateSerializeAssigns(ctx, cur->queryRecord(), selected, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear, serializeOp, serialForm);
                 }
                 else if (needToClear || mustInitializeField(cur))
                 {
@@ -11163,19 +11233,19 @@ void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression *
                 break;
             }
         case no_record:
-            generateSerializeAssigns(ctx, cur, selector, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
+            generateSerializeAssigns(ctx, cur, selector, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear, serializeOp, serialForm);
             break;
         case no_ifblock:
             //Filter on target...
             UNIMPLEMENTED;
-            generateSerializeAssigns(ctx, cur->queryChild(1), selector, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
+            generateSerializeAssigns(ctx, cur->queryChild(1), selector, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear, serializeOp, serialForm);
             break;
         }
     }
 }
 
 
-void HqlCppTranslator::generateSerializeFunction(BuildCtx & ctx, const char * funcName, bool serialize, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects)
+void HqlCppTranslator::generateSerializeFunction(BuildCtx & ctx, const char * funcName, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, node_operator serializeOp, _ATOM serialForm)
 {
     StringBuffer s;
 
@@ -11193,8 +11263,9 @@ void HqlCppTranslator::generateSerializeFunction(BuildCtx & ctx, const char * fu
     IHqlExpression * selfSelect = tgtCursor->querySelector();
     IHqlExpression * record = tgtDataset.queryDataset()->queryRecord();
 
-    generateSerializeAssigns(r2kctx, record, tgtDataset.querySelector(), selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, !isFixedRecordSize(record));
+    generateSerializeAssigns(r2kctx, record, tgtDataset.querySelector(), selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, !isFixedRecordSize(record), serializeOp, serialForm);
 
+    const bool serialize = (serializeOp == no_serialize);
     BoundRow * recordCursor = serialize ? srcCursor : tgtCursor;
     OwnedHqlExpr recordSize = getRecordSize(recordCursor->querySelector());
     OwnedHqlExpr recordSizeVar = createVariable("thisRecordSize", LINK(unsignedType));
@@ -11234,7 +11305,7 @@ void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator
                     break;
                 }
 
-                OwnedHqlExpr serializedField = getSerializedForm(value->queryChild(1));
+                OwnedHqlExpr serializedField = getSerializedForm(value->queryChild(1), diskAtom); // Could be internal, but may require serialized compare
                 OwnedHqlExpr mappedSelect = dataset.mapScalar(value,queryActiveTableSelector());
                 keyFields.append(*LINK(serializedField));
                 keySelects.append(*createSelectExpr(LINK(mappedSelect->queryChild(0)), LINK(serializedField)));
@@ -11274,8 +11345,8 @@ void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator
 
             DatasetReference keyActiveRef(keyDataset, no_activetable, NULL);
 
-            generateSerializeFunction(classctx, "recordToKey", true, dataset, keyActiveRef, datasetSelects, keySelects);
-            generateSerializeFunction(classctx, "keyToRecord", false, keyActiveRef, dataset, keySelects, datasetSelects);
+            generateSerializeFunction(classctx, "recordToKey", dataset, keyActiveRef, datasetSelects, keySelects, no_serialize, diskAtom);
+            generateSerializeFunction(classctx, "keyToRecord", keyActiveRef, dataset, keySelects, datasetSelects, no_deserialize, diskAtom);
             buildMetaMember(classctx, keyRecord, false, "queryRecordSize");
 
             endNestedClass();
@@ -14532,10 +14603,11 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySerialize(BuildCtx & ctx, IHql
     //MORE: I don't have any examples that trigger this code as far as I know...
     _ATOM func = serialize ? rtlSerializeToBuilderAtom : rtlDeserializeToBuilderAtom;
     _ATOM kind = serialize ? serializerAtom : deserializerAtom;
+    _ATOM serializeForm = serialize ? expr->queryChild(2)->queryName() : expr->queryChild(1)->queryName();
 
     IHqlExpression * record = expr->queryRecord();
     HqlExprArray args;
-    args.append(*createRowSerializer(ctx, record, kind));
+    args.append(*createSerializer(ctx, record, serializeForm, kind));
     args.append(*ensureActiveRow(dataset));
     Owned<ITypeInfo> type = makeTransformType(record->getType());
     OwnedHqlExpr call = bindFunctionCall(func, args, type);
@@ -15783,7 +15855,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
     buildCompareClass(instance->nestedctx, "compare", sortlist, DatasetReference(dataset));
 
     IHqlExpression * record = dataset->queryRecord();
-    OwnedHqlExpr serializedRecord = getSerializedForm(record);
+    _ATOM serializeType = diskAtom; //MORE: Does this place a dependency on the implementation?
+    OwnedHqlExpr serializedRecord = getSerializedForm(record, serializeType);
     if (!targetRoxie())
     {
         if (record != serializedRecord)
@@ -15793,7 +15866,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
             OwnedHqlExpr selSeq = createSelectorSequence();
             OwnedHqlExpr leftSelector = createSelector(no_left, dataset, selSeq);
             OwnedHqlExpr mappedSortlist = replaceSelector(sortlist, dataset, leftSelector);
-            OwnedHqlExpr serializedSortlist = replaceMemorySelectorWithSerializedSelector(mappedSortlist, record, no_left, selSeq);
+            OwnedHqlExpr serializedSortlist = replaceMemorySelectorWithSerializedSelector(mappedSortlist, record, no_left, selSeq, serializeType);
             OwnedHqlExpr serializedDataset = createDataset(no_null, LINK(serializedRecord));
             DatasetReference serializedRef(serializedDataset, no_left, selSeq);
             try
@@ -17279,7 +17352,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySetResult(BuildCtx & ctx, IHql
 
     noteResultDefined(ctx, instance, sequence, name, isRoot);
     if (attribute->isDatarow())
-        attribute.setown(::ensureSerialized(attribute));
+        attribute.setown(::ensureSerialized(attribute, diskAtom));
 
     if (kind == TAKremoteresult)
     {
@@ -17723,7 +17796,7 @@ IHqlExpression * HqlCppTranslator::createRecordInheritMaxLength(HqlExprArray & f
         if (!max && hasMaxLength(record))
         {
             //maxlength inherited somewhere...
-            OwnedHqlExpr serializedDonorRecord = getSerializedForm(record);
+            OwnedHqlExpr serializedDonorRecord = getSerializedForm(record, diskAtom);
             ColumnToOffsetMap * map = queryRecordOffsetMap(serializedDonorRecord);
             max.setown(createAttribute(maxLengthAtom, getSizetConstant(map->getMaxSize())));
         }
@@ -17736,7 +17809,7 @@ IHqlExpression * HqlCppTranslator::createRecordInheritMaxLength(HqlExprArray & f
     fields.trunc(prevLength);
     if (max)
     {
-        OwnedHqlExpr serialized = getSerializedForm(ret);
+        OwnedHqlExpr serialized = getSerializedForm(ret, diskAtom);
         if (isFixedRecordSize(serialized))
             ret.setown(createRecord(fields));
     }

ecl/hqlcpp/hqlhtcpp.ipp

@@ -273,8 +273,8 @@ public:
     virtual void setRow(BuildCtx & ctx, IReferenceSelector * rhs);
     virtual IReferenceSelector * select(BuildCtx & ctx, IHqlExpression * selectExpr);
 
-    virtual void buildDeserialize(BuildCtx & ctx, IHqlExpression * helper);
-    virtual void buildSerialize(BuildCtx & ctx, IHqlExpression * helper);
+    virtual void buildDeserialize(BuildCtx & ctx, IHqlExpression * helper, _ATOM serializeForm);
+    virtual void buildSerialize(BuildCtx & ctx, IHqlExpression * helper, _ATOM serializeForm);
 
 private:
     DatasetSelector(DatasetSelector * _parent, BoundRow * _cursor, AColumnInfo * _column, IHqlExpression * _path);

ecl/hqlcpp/hqlinline.cpp

@@ -1505,9 +1505,10 @@ void ClassEvalContext::createMemberAlias(CtxCollection & ctxs, BuildCtx & ctx, I
     assertex(ctxs.evalctx != NULL);
     translator.expandAliases(*ctxs.evalctx, value);
 
+    const _ATOM serializeForm = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
     CHqlBoundTarget tempTarget;
     translator.buildTempExpr(*ctxs.evalctx, ctxs.declarectx, tempTarget, value, FormatNatural, false);
-    ensureSerialized(ctxs, tempTarget);
+    ensureSerialized(ctxs, tempTarget, serializeForm);  // MORE: Check modifying this changes the serialized data
 
     tgt.setFromTarget(tempTarget);
     ctxs.declarectx.associateExpr(value, tgt);
@@ -1524,10 +1525,10 @@ void ClassEvalContext::doCallNestedHelpers(const char * member, const char * act
 }
 
 
-void ClassEvalContext::ensureSerialized(CtxCollection & ctxs, const CHqlBoundTarget & target)
+void ClassEvalContext::ensureSerialized(CtxCollection & ctxs, const CHqlBoundTarget & target, _ATOM serializeForm)
 {
     if (ctxs.serializectx)
-        translator.ensureSerialized(target, *ctxs.serializectx, *ctxs.deserializectx, "*in", "out");
+        translator.ensureSerialized(target, *ctxs.serializectx, *ctxs.deserializectx, "*in", "out", serializeForm);
 }
 
 AliasKind ClassEvalContext::evaluateExpression(BuildCtx & ctx, IHqlExpression * value, CHqlBoundExpr & tgt, bool evaluateLocally)
@@ -1614,7 +1615,8 @@ AliasKind ClassEvalContext::evaluateExpression(BuildCtx & ctx, IHqlExpression *
 
 void ClassEvalContext::tempCompatiablityEnsureSerialized(const CHqlBoundTarget & tgt)
 {
-    ensureSerialized(onCreate, tgt);
+    const _ATOM serializeForm = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
+    ensureSerialized(onCreate, tgt, serializeForm);
 }
 
 bool ClassEvalContext::isRowInvariant(IHqlExpression * expr)

ecl/hqlcpp/hqlinline.hpp

@@ -160,7 +160,7 @@ protected:
     IHqlExpression * cloneExprInClass(CtxCollection & ctxs, IHqlExpression * expr);
     void createMemberAlias(CtxCollection & ctxs, BuildCtx & ctx, IHqlExpression * value, CHqlBoundExpr & tgt);
     void doCallNestedHelpers(const char * member, const char * acticity);
-    void ensureSerialized(CtxCollection & ctxs, const CHqlBoundTarget & tgt);
+    void ensureSerialized(CtxCollection & ctxs, const CHqlBoundTarget & tgt, _ATOM serializeForm);
 
 protected:
     CtxCollection onCreate;

ecl/hqlcpp/hqliproj.cpp

@@ -456,7 +456,7 @@ void UsedFieldSet::calcFinalRecord(bool canPack, bool ignoreIfEmpty)
     if (canPack)
         finalRecord.setown(getPackedRecord(finalRecord));
 
-    OwnedHqlExpr serializedRecord = getSerializedForm(finalRecord);
+    OwnedHqlExpr serializedRecord = getSerializedForm(finalRecord, diskAtom);
     if (maxRecordSizeUsesDefault(serializedRecord))
     {
         HqlExprArray recordFields;
@@ -468,7 +468,7 @@ void UsedFieldSet::calcFinalRecord(bool canPack, bool ignoreIfEmpty)
         else
         {
             bool isKnownSize, useDefaultRecordSize;
-            OwnedHqlExpr oldSerializedRecord = getSerializedForm(originalRecord);
+            OwnedHqlExpr oldSerializedRecord = getSerializedForm(originalRecord, diskAtom);
             unsigned oldRecordSize = getMaxRecordSize(oldSerializedRecord, 0, isKnownSize, useDefaultRecordSize);
             if (!useDefaultRecordSize)
                 recordFields.append(*createAttribute(maxLengthAtom, getSizetConstant(oldRecordSize)));

ecl/hqlcpp/hqllib.cpp

@@ -119,9 +119,10 @@ unsigned HqlCppLibrary::getHash(const HqlExprArray & values, unsigned crc) const
         case type_row:
         case type_table:
         case type_groupedtable:
+        case type_dictionary:
             {
                 OwnedHqlExpr normalizedRecord = normalizeRecord(translator, cur.queryRecord());
-                OwnedHqlExpr serialized = getSerializedForm(normalizedRecord);
+                OwnedHqlExpr serialized = getSerializedForm(normalizedRecord, diskAtom);
                 unsigned recordCrc = getExpressionCRC(serialized);
                 crc = hashc((const byte *)&tc, sizeof(tc), crc);
                 crc = hashc((const byte *)&recordCrc, sizeof(recordCrc), crc);

ecl/hqlcpp/hqlsource.cpp

@@ -279,7 +279,7 @@ static IHqlExpression * createFileposCall(HqlCppTranslator & translator, _ATOM n
 
 void VirtualFieldsInfo::gatherVirtualFields(IHqlExpression * _record, bool ignoreVirtuals, bool ensureSerialized)
 {
-    OwnedHqlExpr record = ensureSerialized ? getSerializedForm(_record) : LINK(_record);
+    OwnedHqlExpr record = ensureSerialized ? getSerializedForm(_record, diskAtom) : LINK(_record);
     if (record != _record)
         requiresDeserialize = true;
 
@@ -494,7 +494,7 @@ static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IH
                 //Simplest would be to move getSerializedForm code + call that first.
                 if (cur->hasProperty(_linkCounted_Atom))
                 {
-                    newField = getSerializedForm(cur);
+                    newField = getSerializedForm(cur, diskAtom);
                     assertex(newField != cur);
                 }
                 else
@@ -1272,7 +1272,7 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
         {
             IHqlExpression * record = expr->queryRecord();
             assertex(fieldInfo.virtualsAtEnd);
-            assertex(!recordRequiresSerialization(record));
+            assertex(!recordRequiresSerialization(record, diskAtom));
             CHqlBoundExpr bound;
             StringBuffer s;
             translator.getRecordSize(ctx, expr, bound);
@@ -1546,10 +1546,10 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
                 //and would mean the roxie/thor code required changing
                 if (translator.targetRoxie())
                 {
-                    OwnedHqlExpr serializedRhsRecord = getSerializedForm(memoryRhsRecord);
+                    OwnedHqlExpr serializedRhsRecord = getSerializedForm(memoryRhsRecord, diskAtom);
                     OwnedHqlExpr serializedRhs = createDataset(no_null, LINK(serializedRhsRecord));
                     rightCursor = translator.bindTableCursor(subctx, serializedRhs, "right", no_right, querySelSeq(expr));
-                    transform.setown(replaceMemorySelectorWithSerializedSelector(transform, memoryRhsRecord, no_right, querySelSeq(expr)));
+                    transform.setown(replaceMemorySelectorWithSerializedSelector(transform, memoryRhsRecord, no_right, querySelSeq(expr), diskAtom));
                 }
                 else
                 {
@@ -7045,7 +7045,7 @@ public:
         selSeq.set(querySelSeq(fetchExpr));
         fetchRhs = fetchExpr->queryChild(1);
         memoryRhsRecord = fetchRhs->queryRecord();
-        serializedRhsRecord.setown(getSerializedForm(memoryRhsRecord));
+        serializedRhsRecord.setown(getSerializedForm(memoryRhsRecord, diskAtom));
     }
 
     virtual void buildMembers(IHqlExpression * expr);

ecl/hqlcpp/hqltcppc.cpp

@@ -914,23 +914,23 @@ void CContainerInfo::buildClear(HqlCppTranslator & translator, BuildCtx & ctx, I
     }
 }
 
-void CContainerInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CContainerInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     ForEachItemIn(idx, children)
     {
         CMemberInfo & cur = children.item(idx);
         Owned<IReferenceSelector> ds = cur.getSelector(ctx, selector);
-        cur.buildDeserialize(translator, ctx, ds, helper);
+        cur.buildDeserialize(translator, ctx, ds, helper, serializeForm);
     }
 }
 
-void CContainerInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CContainerInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     ForEachItemIn(idx, children)
     {
         CMemberInfo & cur = children.item(idx);
         Owned<IReferenceSelector> ds = cur.getSelector(ctx, selector);
-        cur.buildSerialize(translator, ctx, ds, helper);
+        cur.buildSerialize(translator, ctx, ds, helper, serializeForm);
     }
 }
 
@@ -977,7 +977,7 @@ void CContainerInfo::setRow(HqlCppTranslator & translator, BuildCtx & ctx, IRefe
     if (!recordTypesMatch(selector->queryType(), source->queryType()))
         throwError(HQLERR_RecordNotCompatible);
 
-    assertex(!recordRequiresSerialization(column->queryRecord()));
+    assertex(!recordRequiresLinkCount(column->queryRecord()));
     CHqlBoundExpr targetAddress, sourceAddress, length;
     source->buildAddress(ctx, sourceAddress);
 
@@ -1163,7 +1163,7 @@ void CIfBlockInfo::buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IRef
     throwUnexpected();
 }
 
-void CIfBlockInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CIfBlockInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     //MORE: This should really associate offset of the ifblock with the offset of its first child as well.
     CHqlBoundExpr boundSize;
@@ -1178,17 +1178,17 @@ void CIfBlockInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ct
 
     //MORE: This test could be avoided if the first child is *actually* variable length
     ensureTargetAvailable(translator, condctx, selector, CContainerInfo::getTotalMinimumSize());
-    CContainerInfo::buildDeserialize(translator, condctx, selector, helper);
+    CContainerInfo::buildDeserialize(translator, condctx, selector, helper, serializeForm);
 }
 
-void CIfBlockInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CIfBlockInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr cond = selector->queryRootRow()->bindToRow(condition, queryRootSelf());
     CHqlBoundExpr bound;
     translator.buildSimpleExpr(ctx, cond, bound);
     BuildCtx condctx(ctx);
     condctx.addFilter(bound.expr);
-    CContainerInfo::buildSerialize(translator, condctx, selector, helper);
+    CContainerInfo::buildSerialize(translator, condctx, selector, helper, serializeForm);
 }
 
 bool CIfBlockInfo::prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state)
@@ -1446,7 +1446,7 @@ void CColumnInfo::buildClear(HqlCppTranslator & translator, BuildCtx & ctx, IRef
     setColumn(translator, ctx, selector, null);
 }
 
-void CColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     CHqlBoundExpr boundSize;
     OwnedHqlExpr unboundSize = ensureType(buildSizeOfUnbound(translator, ctx, selector), sizetType);
@@ -1455,7 +1455,7 @@ void CColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx
     doBuildDeserialize(translator, ctx, selector, helper, boundSize.expr);
 }
 
-void CColumnInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CColumnInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr sizeOfExpr = createValue(no_sizeof, LINK(sizetType), LINK(selector->queryExpr()));
     CHqlBoundExpr boundSize;
@@ -1645,7 +1645,7 @@ IHqlExpression * CPackedIntColumnInfo::buildSizeOfUnbound(HqlCppTranslator & tra
     return createValue(no_translated, LINK(sizetType), translator.bindTranslatedFunctionCall(getPackedSizeAtom, args));
 }
 
-void CPackedIntColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CPackedIntColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr size = getSizetConstant(queryType()->queryPromotedType()->getSize()+1);    // an over-estimate, but more efficient than working out exactly.
     checkAssignOk(translator, ctx, selector, size, 0);
@@ -1716,7 +1716,7 @@ void CSpecialStringColumnInfo::gatherSize(SizeStruct & target)
 }
 
 
-void CSpecialStringColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CSpecialStringColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr address = getColumnAddress(translator, ctx, selector, sizetType);
     OwnedHqlExpr addressStr = getColumnAddress(translator, ctx, selector, queryType(), sizeof(size32_t));
@@ -1854,7 +1854,7 @@ IHqlExpression * CSpecialVStringColumnInfo::buildSizeOfUnbound(HqlCppTranslator
     return createTranslatedOwned(length);
 }
 
-void CSpecialVStringColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CSpecialVStringColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     BoundRow * row = selector->queryRootRow();
 
@@ -2022,7 +2022,7 @@ IHqlExpression * CAlienColumnInfo::buildSizeOfUnbound(HqlCppTranslator & transla
     return doBuildSizeOfUnbound(translator, ctx, selector, NULL);
 }
 
-void CAlienColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CAlienColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr unboundSize = doBuildSizeOfUnbound(translator, ctx, selector, helper);
         
@@ -2217,7 +2217,7 @@ void CBitfieldContainerInfo::buildClear(HqlCppTranslator & translator, BuildCtx
     setColumn(translator, condctx, selector, null);
 }
 
-void CBitfieldContainerInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CBitfieldContainerInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr size = getSizetConstant(column->queryType()->getSize());
     doBuildDeserialize(translator, ctx, selector, helper, size);
@@ -2709,7 +2709,7 @@ IHqlExpression * CXmlColumnInfo::getXmlDatasetExpr(HqlCppTranslator & translator
     //Create the builder for generating a temporary set.
     IHqlExpression * record = expr->queryRecord();
     Owned<IHqlCppDatasetBuilder> builder;
-    if (recordRequiresSerialization(expr->queryRecord()) || translator.queryOptions().tempDatasetsUseLinkedRows)
+    if (recordRequiresLinkCount(expr->queryRecord()) || translator.queryOptions().tempDatasetsUseLinkedRows)
         builder.setown(translator.createLinkedDatasetBuilder(record));
     else
         builder.setown(translator.createBlockedDatasetBuilder(record));
@@ -3452,7 +3452,7 @@ IHqlExpression * SerializationRow::addSerializedValue(IHqlExpression * path, ITy
     _ATOM name = NULL;
     if (path->getOperator() == no_select)
         name = path->queryChild(1)->queryName();
-    Owned<ITypeInfo> newType = getSimplifiedType(type, isConditional, (colocal == NULL));
+    Owned<ITypeInfo> newType = getSimplifiedType(type, isConditional, (colocal == NULL), internalAtom);
     OwnedHqlExpr newSelect = createField(name, newType);
 
     OwnedHqlExpr deserialized;
@@ -3463,9 +3463,14 @@ IHqlExpression * SerializationRow::addSerializedValue(IHqlExpression * path, ITy
     }
     else
     {
-        OwnedHqlExpr srcValue = ::ensureSerialized(path);
+        OwnedHqlExpr srcValue = ::ensureSerialized(path, internalAtom);
         extractBuilder->buildAssign(newSelect, srcValue);
-        deserialized.setown(ensureDeserialized(newSelect, type));
+
+        Linked<ITypeInfo> evaluateType = type;
+        if (evaluateType->getTypeCode() == type_dictionary)
+            evaluateType.setown(setLinkCountedAttr(evaluateType, true));
+
+        deserialized.setown(ensureDeserialized(newSelect, evaluateType, internalAtom));
         if (deserialized != newSelect)
             deserialized.setown(createAlias(deserialized, NULL));           // force it to be evaluated once per start
     }

ecl/hqlcpp/hqltcppc.ipp

@@ -173,8 +173,8 @@ public:
 // AColumnInfo
     virtual void buildSizeOf(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
     virtual void buildClear(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, int direction);
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
-    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
+    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
     virtual IHqlExpression * createSelectorExpr();
@@ -244,8 +244,8 @@ public:
 //AColumnInfo
     virtual void buildAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
     virtual void buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
-    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
+    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
@@ -276,8 +276,8 @@ public:
     virtual void buildSizeOf(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
     virtual void buildClear(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, int direction);
     virtual void buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
-    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
+    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
@@ -294,9 +294,8 @@ protected:
     virtual void buildColumnAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
     virtual void calcCurrentOffset(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
 
-    void buildDeserializeToBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IHqlCppDatasetBuilder * builder, IReferenceSelector * selector, IHqlExpression * helper);
-    void buildDeserializeDatasetUsingBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
-    virtual void buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper);
+    void buildDeserializeToBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IHqlCppDatasetBuilder * builder, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
+    virtual void buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
 };
 
 class HQLCPP_API CSpecialIntColumnInfo : public CColumnInfo
@@ -317,7 +316,7 @@ public:
 //AColumnInfo
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
 
@@ -333,7 +332,7 @@ public:
 
 //AColumnInfo
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
@@ -349,7 +348,7 @@ public:
 //AColumnInfo
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
 
@@ -367,7 +366,7 @@ public:
 //Virtuals for column implementation
     virtual void buildAddress(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
@@ -398,7 +397,7 @@ public:
     virtual void buildAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
     virtual void buildClear(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, int direction);
     virtual void buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
 
@@ -637,7 +636,7 @@ public:
 
 //AColumnInfo
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
@@ -653,8 +652,9 @@ public:
 
 //AColumnInfo
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
+    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
     virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
 
@@ -673,7 +673,7 @@ public:
 
 //AColumnInfo
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
@@ -685,7 +685,7 @@ public:
 
 protected:
     void setColumnFromBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlCppDatasetBuilder * builder);
-    virtual void buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
 
 protected:
     OwnedHqlExpr countField;
@@ -701,8 +701,8 @@ public:
 
 //AColumnInfo
     virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
-    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
-    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
+    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
+    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm);
     virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
     virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
     virtual bool modifyColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value, node_operator op);

ecl/hqlcpp/hqltcppc2.cpp

@@ -74,7 +74,7 @@ IHqlExpression * CChildSetColumnInfo::buildSizeOfUnbound(HqlCppTranslator & tran
     return createValue(no_translated, LINK(sizetType), adjustValue(boundSize, sizeof(bool)+sizeof(size32_t)));
 }
 
-void CChildSetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CChildSetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr address = getColumnAddress(translator, ctx, selector, boolType, 0);
     OwnedHqlExpr addressSize = getColumnAddress(translator, ctx, selector, sizetType, sizeof(bool));
@@ -227,7 +227,7 @@ CChildDatasetColumnInfo::CChildDatasetColumnInfo(CContainerInfo * _container, CM
     ColumnToOffsetMap * offsetMap = map.queryMapping(column->queryRecord(), defaultMaxRecordSize);
     maxChildSize = offsetMap->getMaxSize();
 #ifdef _DEBUG
-    assertex(!recordRequiresSerialization(column->queryRecord()));
+    assertex(!recordRequiresSerialization(column->queryRecord(), internalAtom));
 #endif
 }
 
@@ -245,7 +245,7 @@ void CChildDatasetColumnInfo::gatherSize(SizeStruct & target)
     addVariableSize(sizeof(size32_t), target);
 }
 
-void CColumnInfo::buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CColumnInfo::buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr endMarker = loopctx.getTempDeclare(sizetType, NULL);
     HqlExprArray args;
@@ -259,41 +259,31 @@ void CColumnInfo::buildDeserializeChildLoop(HqlCppTranslator & translator, Build
     loopctx.addLoop(loopCall, NULL, false);
 }
 
-void CColumnInfo::buildDeserializeToBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IHqlCppDatasetBuilder * builder, IReferenceSelector * selector, IHqlExpression * helper)
+void CColumnInfo::buildDeserializeToBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IHqlCppDatasetBuilder * builder, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     BuildCtx loopctx(ctx);
-    buildDeserializeChildLoop(translator, loopctx, selector, helper);
+    buildDeserializeChildLoop(translator, loopctx, selector, helper, serializeForm);
 
-    BoundRow * selfRow = builder->buildDeserializeRow(loopctx, helper);
+    BoundRow * selfRow = builder->buildDeserializeRow(loopctx, helper, serializeForm);
     builder->finishRow(loopctx, selfRow);
 }
 
 
-void CColumnInfo::buildDeserializeDatasetUsingBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
-{
-    Owned<IHqlCppDatasetBuilder> builder = translator.createBlockedDatasetBuilder(column->queryRecord());
-    builder->buildDeclare(ctx);
-
-    buildDeserializeToBuilder(translator, ctx, builder, selector, helper);
-
-    CHqlBoundExpr bound;
-    builder->buildFinish(ctx, bound);
-
-    OwnedHqlExpr translated = bound.getTranslatedExpr();
-    setColumn(translator, ctx, selector, translated);
-}
-
-
-void CChildDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CChildDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     IHqlExpression * record = column->queryRecord();
-
-    if (recordRequiresSerialization(record))
+    assertex(!recordRequiresLinkCount(record)); // Why would it?
+    if (column->isDictionary())
     {
-        buildDeserializeDatasetUsingBuilder(translator, ctx, selector, helper);
-        return;
+        if (serializeForm == diskAtom)
+        {
+            //If we ever generate the meta definition for an internal serialization format then the following needs to be implemented
+            UNIMPLEMENTED_X("deserialize serialized dictionary from disk");
+            return;
+        }
     }
 
+
     if (isConditional())
         checkAssignOk(translator, ctx, selector, queryZero(), sizeof(size32_t));
 
@@ -317,6 +307,22 @@ void CChildDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translator, Bu
 }
 
 
+void CChildDatasetColumnInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
+{
+    if (column->isDictionary())
+    {
+        if (serializeForm == diskAtom)
+        {
+            //If we ever generate the meta definition for an internal serialization format then the following needs to be implemented
+            UNIMPLEMENTED_X("deserialize serialized dictionary from disk");
+        }
+    }
+
+    CColumnInfo::buildSerialize(translator, ctx, selector, helper, serializeForm);
+}
+
+
+
 bool CChildDatasetColumnInfo::buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state)
 {
     OwnedHqlExpr sizeOfDataset = callDeserializerGetSize(translator, ctx, state.helper);
@@ -344,7 +350,7 @@ void CChildDatasetColumnInfo::setColumn(HqlCppTranslator & translator, BuildCtx
     OwnedHqlExpr value = LINK(_value); //ensureExprType(_value, columnType);
     ITypeInfo * valueType = value->queryType();
 
-    assertex(recordTypesMatch(valueType, columnType));
+    assertRecordTypesMatch(valueType, columnType);
 
     bool assignInline = false;  // canEvaluateInline(value);   // MORE: What is the test
 //  bool assignInline = canAssignInline(&ctx, value) && !canEvaluateInline(&ctx, value);
@@ -482,7 +488,7 @@ bool CChildLimitedDatasetColumnInfo::isFixedSize()
     return false;   //MORE:
 }
 
-void CChildLimitedDatasetColumnInfo::buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CChildLimitedDatasetColumnInfo::buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
     OwnedHqlExpr mappedCount = replaceSelector(countField, querySelfReference(), selector->queryExpr()->queryChild(0));
     CHqlBoundExpr bound;
@@ -562,8 +568,9 @@ bool CChildLimitedDatasetColumnInfo::buildReadAhead(HqlCppTranslator & translato
 }
 
 
-void CChildLimitedDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CChildLimitedDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeForm)
 {
+    assertex(!column->isDictionary());
     if (sizeField || !countField)
     {
         ctx.addQuoted("rtlFailUnexpected();");
@@ -572,12 +579,12 @@ void CChildLimitedDatasetColumnInfo::buildDeserialize(HqlCppTranslator & transla
 
     //NB: The serialized form of a dataset with an external count is not the same as a normal dataset
     IHqlExpression * record = column->queryRecord();
-    if (recordRequiresSerialization(record) || !translator.isFixedRecordSize(record))
+    if (recordRequiresSerialization(record, serializeForm) || !translator.isFixedRecordSize(record))
     {
         Owned<IHqlCppDatasetBuilder> builder = translator.createBlockedDatasetBuilder(column->queryRecord());
         builder->buildDeclare(ctx);
 
-        buildDeserializeToBuilder(translator, ctx, builder, selector, helper);
+        buildDeserializeToBuilder(translator, ctx, builder, selector, helper, serializeForm);
 
         CHqlBoundExpr bound;
         builder->buildFinish(ctx, bound);
@@ -585,7 +592,7 @@ void CChildLimitedDatasetColumnInfo::buildDeserialize(HqlCppTranslator & transla
         setColumnFromBuilder(translator, ctx, selector, builder);
     }
     else
-        CColumnInfo::buildDeserialize(translator, ctx, selector, helper);
+        CColumnInfo::buildDeserialize(translator, ctx, selector, helper, serializeForm);
 }
 
 
@@ -665,7 +672,7 @@ IHqlExpression * CChildLinkedDatasetColumnInfo::buildSizeOfUnbound(HqlCppTransla
     return getSizetConstant(sizeof(size32_t) + sizeof(byte * *));
 }
 
-void CChildLinkedDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CChildLinkedDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeFormat)
 {
     if (isConditional())
         checkAssignOk(translator, ctx, selector, queryZero(), sizeof(size32_t) + sizeof(byte * *));
@@ -678,10 +685,26 @@ void CChildLinkedDatasetColumnInfo::buildDeserialize(HqlCppTranslator & translat
     boundTarget.count.setown(convertAddressToValue(addressSize, sizetType));
     boundTarget.expr.setown(convertAddressToValue(addressData, queryType()));
 
+    _ATOM func = NULL;
     HqlExprArray args;
-    args.append(*translator.createRowSerializer(ctx, record, deserializerAtom));
+    args.append(*translator.createSerializer(ctx, record, serializeFormat, deserializerAtom));
+    if (column->isDictionary())
+    {
+        if (serializeFormat == diskAtom)
+        {
+            func = deserializeChildDictionaryFromDatasetFromStreamAtom;
+            StringBuffer lookupHelperName;
+            translator.buildDictionaryHashClass(record, lookupHelperName);
+            args.append(*createQuoted(lookupHelperName.str(), makeBoolType()));
+        }
+        else
+            func = deserializeChildDictionaryFromStreamAtom;
+    }
+    else
+        func = deserializeChildRowsetFromStreamAtom;
+
     args.append(*LINK(helper));
-    OwnedHqlExpr call = translator.bindFunctionCall(deserializerRowsetHelperAtom, args, queryType());
+    OwnedHqlExpr call = translator.bindFunctionCall(func, args, queryType());
     translator.buildExprAssign(ctx, boundTarget, call);
 }
 
@@ -695,15 +718,25 @@ bool CChildLinkedDatasetColumnInfo::buildReadAhead(HqlCppTranslator & translator
 
 
 
-void CChildLinkedDatasetColumnInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper)
+void CChildLinkedDatasetColumnInfo::buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, _ATOM serializeFormat)
 {
     IHqlExpression * record = column->queryRecord();
 
+    _ATOM func = NULL;
     HqlExprArray args;
     args.append(*LINK(helper));
-    args.append(*translator.createRowSerializer(ctx, record, serializerAtom));
+    args.append(*translator.createSerializer(ctx, record, serializeFormat, serializerAtom));
     args.append(*LINK(selector->queryExpr()));
-    OwnedHqlExpr call = translator.bindTranslatedFunctionCall(serializerRowsetHelperAtom, args);
+    if (column->isDictionary())
+    {
+        if (serializeFormat == diskAtom)
+            func = serializeChildDictionaryToDatasetToStreamAtom;
+        else
+            func = serializeChildDictionaryToStreamAtom;
+    }
+    else
+        func = serializeChildRowsetToStreamAtom;
+    OwnedHqlExpr call = translator.bindTranslatedFunctionCall(func, args);
     translator.buildStmt(ctx, call);
 }
 
@@ -742,7 +775,7 @@ void CChildLinkedDatasetColumnInfo::setColumn(HqlCppTranslator & translator, Bui
     LinkedHqlExpr value = _value;
     ITypeInfo * valueType = value->queryType();
 
-    assertex(recordTypesMatch(valueType, resultType));
+    assertRecordTypesMatch(resultType, valueType);
 
     value.setown(addDatasetLimits(translator, ctx, selector, value));
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -7289,7 +7289,7 @@ IHqlExpression * NewScopeMigrateTransformer::createTransformed(IHqlExpression *
                         }
                         if (rowOp == no_createrow)
                             break;
-                        if (!isInlineTrivialDataset(row) && !isContextDependent(row) && !transformed->isDataset())
+                        if (!isInlineTrivialDataset(row) && !isContextDependent(row) && !transformed->isDataset() && !transformed->isDictionary())
                         {
                             if (isIndependentOfScope(row))
                                 return hoist(expr, transformed);
@@ -8460,7 +8460,7 @@ IHqlExpression * HqlLinkedChildRowTransformer::ensureInputSerialized(IHqlExpress
 {
     LinkedHqlExpr dataset = expr->queryChild(0);
     IHqlExpression * record = dataset->queryRecord();
-    OwnedHqlExpr serializedRecord = getSerializedForm(record);
+    OwnedHqlExpr serializedRecord = getSerializedForm(record, diskAtom);
 
     //If the dataset requires serialization, it is much more efficient to serialize before the sort, than to serialize after.
     if (record == serializedRecord)
@@ -8745,7 +8745,7 @@ IHqlExpression * HqlScopeTagger::transformSelect(IHqlExpression * expr)
     IHqlExpression * field = expr->queryChild(1);
     if (ds->isDataset())
     {
-        if (!expr->isDataset() && !expr->isDatarow())
+        if (!expr->isDataset() && !expr->isDatarow() && !expr->isDictionary())
         {
             //If the left is a dataset, and the right isn't a dataset or a datarow then this doesn't make sense - it is an illegal
             return createSelectExpr(newDs.getClear(), LINK(field));

ecl/hthor/hthor.cpp

@@ -485,9 +485,9 @@ void CHThorDiskWriteActivity::open()
     // Open an output file...
     file.setown(createIFile(filename));
     inputMeta.set(input->queryOutputMeta());
-    serializedOutputMeta.set(input->queryOutputMeta()->querySerializedMeta());//returns outputMeta if serialization not needed
+    serializedOutputMeta.set(input->queryOutputMeta()->querySerializedDiskMeta());//returns outputMeta if serialization not needed
 
-    Linked<IRecordSize> groupedMeta = input->queryOutputMeta()->querySerializedMeta();
+    Linked<IRecordSize> groupedMeta = input->queryOutputMeta()->querySerializedDiskMeta();
     if(grouped)
         groupedMeta.setown(createDeltaRecordSize(groupedMeta, +1));
     blockcompressed = checkIsCompressed(helper.getFlags(), serializedOutputMeta.getFixedSize(), grouped);//TDWnewcompress for new compression, else check for row compression
@@ -1262,7 +1262,7 @@ public:
     {
         groupSignalled = true; // i.e. don't start with a NULL row
         CHThorSimpleActivityBase::ready();
-        rowDeserializer.setown(rowAllocator->createRowDeserializer(agent.queryCodeContext()));
+        rowDeserializer.setown(rowAllocator->createDiskDeserializer(agent.queryCodeContext()));
         readTransformer.setown(createReadRowStream(rowAllocator, rowDeserializer, helper.queryXmlTransformer(), helper.queryCsvTransformer(), helper.queryXmlIteratorPath(), helper.getPipeFlags()));
         openPipe(helper.getPipeProgram());
     }
@@ -1432,8 +1432,8 @@ public:
         // From the create() in roxie
 
         inputMeta.set(input->queryOutputMeta());
-        rowSerializer.setown(inputMeta.createRowSerializer(agent.queryCodeContext(), activityId));
-        rowDeserializer.setown(rowAllocator->createRowDeserializer(agent.queryCodeContext()));
+        rowSerializer.setown(inputMeta.createDiskSerializer(agent.queryCodeContext(), activityId));
+        rowDeserializer.setown(rowAllocator->createDiskDeserializer(agent.queryCodeContext()));
         writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
 
         // From the start() in roxie
@@ -1613,7 +1613,7 @@ public:
     {
         CHThorActivityBase::ready();
         inputMeta.set(input->queryOutputMeta());
-        rowSerializer.setown(inputMeta.createRowSerializer(agent.queryCodeContext(), activityId));
+        rowSerializer.setown(inputMeta.createDiskSerializer(agent.queryCodeContext(), activityId));
         writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
 
         firstRead = true;
@@ -5785,8 +5785,8 @@ void CHThorWorkUnitWriteActivity::execute()
     __int64 initialRows = rows;
 
     Owned<IOutputRowSerializer> rowSerializer;
-    if (input->queryOutputMeta()->getMetaFlags() & MDFneedserialize)
-        rowSerializer.setown( input->queryOutputMeta()->createRowSerializer(agent.queryCodeContext(), activityId) );
+    if (input->queryOutputMeta()->getMetaFlags() & MDFneedserializedisk)
+        rowSerializer.setown( input->queryOutputMeta()->createDiskSerializer(agent.queryCodeContext(), activityId) );
 
     int seq = helper.getSequence();
     bool toStdout = (seq >= 0) && agent.queryWriteResultsToStdout();
@@ -5914,7 +5914,7 @@ void CHThorDictionaryWorkUnitWriteActivity::execute()
     size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt("outputLimit", defaultWorkUnitWriteLimit) * 0x100000;
     MemoryBuffer rowdata;
     CThorDemoRowSerializer out(rowdata);
-    Owned<IOutputRowSerializer> serializer = input->queryOutputMeta()->createRowSerializer(agent.queryCodeContext(), activityId);
+    Owned<IOutputRowSerializer> serializer = input->queryOutputMeta()->createDiskSerializer(agent.queryCodeContext(), activityId);
     rtlSerializeDictionary(out, serializer, builder.getcount(), builder.queryrows());
     if(outputLimit && (rowdata.length()  > outputLimit))
     {
@@ -6619,7 +6619,7 @@ void CHThorWorkunitReadActivity::ready()
 {
     CHThorSimpleActivityBase::ready();
 
-    rowDeserializer.setown(rowAllocator->createRowDeserializer(agent.queryCodeContext()));
+    rowDeserializer.setown(rowAllocator->createDiskDeserializer(agent.queryCodeContext()));
 
     if(first)
     {
@@ -7427,7 +7427,7 @@ void CHThorRawIteratorActivity::ready()
     helper.queryDataset(lenData, data);
     resultBuffer.setBuffer(lenData, const_cast<void *>(data), false);
     eogPending = false;
-    rowDeserializer.setown(rowAllocator->createRowDeserializer(agent.queryCodeContext()));
+    rowDeserializer.setown(rowAllocator->createDiskDeserializer(agent.queryCodeContext()));
 }
 
 void CHThorRawIteratorActivity::done()
@@ -7782,7 +7782,7 @@ void CHThorDiskReadBaseActivity::gatherInfo(IFileDescriptor * fileDesc)
         grouped = ((helper.getFlags() & TDXgrouped) != 0);
     }
 
-    diskMeta.set(helper.queryDiskRecordSize()->querySerializedMeta());
+    diskMeta.set(helper.queryDiskRecordSize()->querySerializedDiskMeta());
     if (grouped)
         diskMeta.setown(new CSuffixedOutputMeta(+1, diskMeta));
     if (outputMeta.isFixedSize())
@@ -8081,8 +8081,8 @@ void CHThorBinaryDiskReadBase::ready()
         diskMeta.set(outputMeta);
     segMonitors.kill();
     segHelper.createSegmentMonitors(this);
-    prefetcher.setown(diskMeta->createRowPrefetcher(agent.queryCodeContext(), activityId));
-    deserializer.setown(diskMeta->createRowDeserializer(agent.queryCodeContext(), activityId));
+    prefetcher.setown(diskMeta->createDiskPrefetcher(agent.queryCodeContext(), activityId));
+    deserializer.setown(diskMeta->createDiskDeserializer(agent.queryCodeContext(), activityId));
 }
 
 bool CHThorBinaryDiskReadBase::openNext()
@@ -9252,10 +9252,12 @@ public:
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
     virtual void destruct(byte * self)  {}
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputMetaData * querySerializedMeta() { return this; }
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
 };
 

ecl/hthor/hthor.ipp

@@ -301,7 +301,7 @@ protected:
     virtual void setFormat(IFileDescriptor * desc);
     virtual bool isFixedWidth() 
     { 
-        return (input->queryOutputMeta()->querySerializedMeta()->isFixedSize());
+        return (input->queryOutputMeta()->querySerializedDiskMeta()->isFixedSize());
     }
 
     void resolve();

ecl/hthor/hthorkey.cpp

@@ -2368,7 +2368,7 @@ public:
     {
         CHThorFetchActivityBase::ready();
         rowLimit = helper.getRowLimit();
-        rowDeserializer.setown(helper.queryDiskRecordSize()->createRowDeserializer(agent.queryCodeContext(), activityId));
+        rowDeserializer.setown(helper.queryDiskRecordSize()->createDiskDeserializer(agent.queryCodeContext(), activityId));
         diskAllocator.setown(agent.queryCodeContext()->getRowAllocator(helper.queryDiskRecordSize(), activityId));
     }
 
@@ -2527,7 +2527,7 @@ public:
     {
         CHThorFetchActivityBase::ready();
         rowLimit = helper.getRowLimit();
-        rowDeserializer.setown(helper.queryDiskRecordSize()->createRowDeserializer(agent.queryCodeContext(), activityId));
+        rowDeserializer.setown(helper.queryDiskRecordSize()->createDiskDeserializer(agent.queryCodeContext(), activityId));
     }
 
     virtual void onLimitExceeded()
@@ -3412,7 +3412,7 @@ public:
         }
         if (needsDiskRead)
         {
-            rowDeserializer.setown(helper.queryDiskRecordSize()->createRowDeserializer(agent.queryCodeContext(), activityId));
+            rowDeserializer.setown(helper.queryDiskRecordSize()->createDiskDeserializer(agent.queryCodeContext(), activityId));
             diskAllocator.setown(agent.queryCodeContext()->getRowAllocator(helper.queryDiskRecordSize(), activityId));
         }
     }

ecl/regress/modules/serialtest.ecl

@@ -0,0 +1,61 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+EXPORT serialTest := MODULE
+
+EXPORT wordRec := { string word; };
+
+//A DATASET nested two levels deep
+EXPORT bookDsRec := RECORD
+  string title;
+  DATASET(wordRec) words;
+END;
+
+EXPORT libraryDsRec := RECORD
+  string owner;
+  DATASET(bookDsRec) books;
+END;
+
+
+// Same for a DICTIONARY
+EXPORT bookDictRec := RECORD
+  string title
+  =>
+  DICTIONARY(wordRec) words;
+END;
+
+EXPORT libraryDictRec := RECORD
+  string owner
+  =>
+  DICTIONARY(bookDictRec) books;
+END;
+
+EXPORT DsFilename := 'REGRESS::SerialLibraryDs';
+
+EXPORT DictFilename := 'REGRESS::SerialLibraryDict';
+
+EXPORT DictKeyFilename := 'REGRESS::SerialLibraryKeyDict';
+
+EXPORT BookKeyFilename := 'REGRESS::SerialBookKey';
+
+EXPORT libraryDictionaryFile := DATASET(DictFilename, LibraryDictRec, THOR);
+
+EXPORT libraryDatasetFile := DATASET(DsFilename, LibraryDsRec, THOR);
+
+EXPORT bookIndex := INDEX({ string20 title }, { dataset(wordRec) words }, BookKeyFilename);
+
+END; /* serial */

ecl/regress/serial1.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Create a datafile containing child and grandchild datasets
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+ 
+libraryDs := DATASET([
+    { 'gavin',
+        [{'the hobbit',
+            [{'gandalf'},{'rivendell'},{'dragon'},{'dwarves'},{'elves'}]},
+         {'eragon',
+            [{'eragon'},{'dragon'},{'spine'},{'elves'},{'dwarves'},{'krull'}]}
+        ]},
+    { 'jim',
+        [{'complete diy',
+            [{'heating'},{'electrics'},{'nuclear reactors'},{'spaceships'}]},
+        {'cheeses',
+            [{'cheddar'},{'parmesan'},{'stilton'},{'wensleydale'}]}
+        ]}], SerialTest.libraryDsRec);
+        
+OUTPUT(libraryDs,,SerialTest.DsFilename,OVERWRITE);

ecl/regress/serial2.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Create a datafile containing child and grandchild dictionaries
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+libraryDs := DATASET([
+    { 'gavin' =>
+        [{'the hobbit' =>
+            [{'gandalf'},{'rivendell'},{'dragon'},{'dwarves'},{'elves'}]},
+         {'eragon' =>
+            [{'eragon'},{'dragon'},{'spine'},{'elves'},{'dwarves'},{'krull'}]}
+        ]},
+    { 'jim' =>
+        [{'complete diy' =>
+            [{'heating'},{'electrics'},{'nuclear reactors'},{'spaceships'}]},
+        {'cheeses' =>
+            [{'cheddar'},{'parmesan'},{'stilton'},{'wensleydale'}]}
+        ]}], SerialTest.libraryDictRec);
+        
+OUTPUT(libraryDs,,SerialTest.DictFilename,OVERWRITE);

ecl/regress/serial2b.ecl

@@ -0,0 +1,41 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Create an index containing child and grandchild dictionaries
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+libraryDs := DATASET([
+    { 'gavin' =>
+        [{'the hobbit' =>
+            [{'gandalf'},{'rivendell'},{'dragon'},{'dwarves'},{'elves'}]},
+         {'eragon' =>
+            [{'eragon'},{'dragon'},{'spine'},{'elves'},{'dwarves'},{'krull'}]}
+        ]},
+    { 'jim' =>
+        [{'complete diy' =>
+            [{'heating'},{'electrics'},{'nuclear reactors'},{'spaceships'}]},
+        {'cheeses' =>
+            [{'cheddar'},{'parmesan'},{'stilton'},{'wensleydale'}]}
+        ]}], SerialTest.libraryDictRec);
+
+libraryIndex := INDEX(libraryDs, { (string20)owner }, { DICTIONARY(SerialTest.bookDictRec) books := books }, SerialTest.DictKeyFilename);
+//libraryIndex := INDEX(libraryDs, { (string20)owner }, { libraryDs }, SerialTest.DictKeyFilename);
+
+BUILD(libraryIndex);

ecl/regress/serial2c.ecl

@@ -0,0 +1,42 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Create an index containing a child datasets
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+libraryDs := DATASET([
+    { 'gavin',
+        [{'the hobbit',
+            [{'gandalf'},{'rivendell'},{'dragon'},{'dwarves'},{'elves'}]},
+         {'eragon',
+            [{'eragon'},{'dragon'},{'spine'},{'elves'},{'dwarves'},{'krull'}]}
+        ]},
+    { 'jim',
+        [{'complete diy',
+            [{'heating'},{'electrics'},{'nuclear reactors'},{'spaceships'}]},
+        {'cheeses',
+            [{'cheddar'},{'parmesan'},{'stilton'},{'wensleydale'}]}
+        ]}], SerialTest.libraryDsRec);
+
+allBooks := libraryDs.books;
+
+bookIndex := INDEX(allBooks, { string20 title := title }, { dataset(SerialTest.wordRec) words := words }, SerialTest.BookKeyFilename);
+
+BUILD(bookIndex, overwrite);

ecl/regress/serial3a.ecl

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test reading a file containg a child and grand-child dataset with a dataset definition
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDsRec, THOR);
+output(TABLE(inDs, { owner, COUNT(books) }));

ecl/regress/serial3b.ecl

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test reading a file containg a child and grand-child *dictionary* with a dataset definition
+
+#option ('skipFileFormatCrcCheck', true);
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DictFilename, SerialTest.LibraryDsRec, THOR);
+output(TABLE(inDs, { owner, COUNT(books) }));

ecl/regress/serial3c.ecl

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test reading a file containg a child and grand-child *dataset* with a dictionary definition
+
+#option ('skipFileFormatCrcCheck', true);
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDictRec, THOR);
+output(TABLE(inDs, { owner, COUNT(books) }));

ecl/regress/serial3d.ecl

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test reading a file containg a child and grand-child dictionary with a dictionary definition
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DictFilename, SerialTest.LibraryDictRec, THOR);
+output(TABLE(inDs, { owner, COUNT(books) }));

ecl/regress/serial4a.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test spilling a file containg child and grand-child dictionary
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDsRec, THOR);
+
+sort1 := NOFOLD(SORT(inDs, owner));
+
+//Two sorts will cause a spill in thor, two outputs will cause one in hthor.
+//We really should have a SPILL() activity which introduces one.
+sort2 := SORT(sort1, books[1].title);
+
+output(sort1);
+output(sort2);

ecl/regress/serial4b.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test spilling a file containg child and grand-child dictionary
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DictFilename, SerialTest.LibraryDictRec, THOR);
+
+sort1 := NOFOLD(SORT(inDs, owner));
+
+//Two sorts will cause a spill in thor, two outputs will cause one in hthor.
+//We really should have a SPILL() activity which introduces one.
+sort2 := SORT(sort1, DATASET(books)[1].title);
+
+output(sort1);
+output(sort2);

ecl/regress/serial5a.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDsRec, THOR);
+
+output(JOIN(inDs.books, SerialTest.bookIndex, LEFT.title = RIGHT.title));

ecl/regress/serial5b.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDsRec, THOR);
+
+output(JOIN(inDs, SerialTest.bookIndex, KEYED(RIGHT.title IN SET(LEFT.books, title))));

ecl/regress/serial5c.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDictRec, THOR);
+
+output(JOIN(inDs, SerialTest.bookIndex, KEYED(RIGHT.title IN LEFT.books)));

ecl/regress/serial6a.ecl

@@ -0,0 +1,30 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test serializing a dataset to the slave for a remote keyed join
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+
+inDs := SerialTest.libraryDatasetFile;
+
+output(JOIN(inDs.books, SerialTest.bookIndex, 
+                LEFT.title = RIGHT.title AND
+                 RIGHT.words[1].word IN SET(LEFT.words, word)
+                 ));

ecl/regress/serial6b.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test serializing a dictionary to the slave for a remote keyed join
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDsRec, THOR);
+
+output(JOIN(inDs.books, SerialTest.bookIndex, 
+                LEFT.title = RIGHT.title AND
+                 RIGHT.words[1].word NOT IN SET(LEFT.words, word)
+                 ));

ecl/regress/serial6c.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := SerialTest.libraryDictionaryFile;
+
+books := DATASET(inDs.books);
+
+output(JOIN(books, SerialTest.bookIndex, 
+                LEFT.title = RIGHT.title AND
+                 RIGHT.words[1].word IN LEFT.words
+                 ));

ecl/regress/serial6d.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+inDs := SerialTest.libraryDictionaryFile;
+
+books := NORMALIZE(inDs, DATASET(LEFT.books), transform(RIGHT));
+
+output(JOIN(books, SerialTest.bookIndex, 
+                LEFT.title = RIGHT.title AND
+                 RIGHT.words[1].word IN LEFT.words
+                 ));

ecl/regress/serial7a.ecl

@@ -0,0 +1,23 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec);
+
+output(SerialTest.bookIndex(WILD(title), EXISTS(words(word IN interestingWords))));
+

ecl/regress/serial7a2_error.ecl

@@ -0,0 +1,27 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Error: The reference to words.word in the OUTPUT should complain that "words" is not in scope
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec);
+
+output(SerialTest.bookIndex(WILD(title), words.word IN interestingWords));
+

ecl/regress/serial7a_error.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Error: The reference to words.word in the OUTPUT should complain that "words" is not in scope
+//Unfortunately it gets optimized away by EXISTS(x) always being true.  Check a warning is output.
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec);
+
+output(SerialTest.bookIndex(WILD(title), EXISTS(words.word IN interestingWords)));
+

ecl/regress/serial7b.ecl

@@ -0,0 +1,23 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec) : global;
+
+output(SerialTest.bookIndex(WILD(title), EXISTS(words(word IN interestingWords))));
+

ecl/regress/serial7c.ecl

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec) : global(few);
+
+output(SerialTest.bookIndex(WILD(title), EXISTS(words(word IN interestingWords))));
+

ecl/regress/serial7d.ecl

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec) : once; // Shouldn't need to serialize to slaves in roxie.
+
+output(SerialTest.bookIndex(WILD(title), EXISTS(words(word IN interestingWords))));
+

ecl/regress/serial8a.ecl

@@ -0,0 +1,30 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test that dictionaries are correctly serialized to colocal child queries
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec);
+
+inDs := SerialTest.libraryDatasetFile;
+
+p := TABLE(inDs, { owner, DATASET(SerialTest.bookDsrec) books := SORT(books, title)(EXISTS(words(word in interestingWords)))});
+
+output(p);

ecl/regress/serial8b.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test that dictionaries are correctly serialized to nonlocal child queries
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec);
+
+inDs := SerialTest.libraryDatasetFile;
+
+filteredBooks(string searchName, dictionary(SerialTest.wordRec) searchWords) := 
+            SerialTest.bookIndex(
+                KEYED(title = searchName), 
+                EXISTS(words(word IN searchWords))
+            );
+
+
+p := TABLE(inDs, { owner, DATASET(recordof(SerialTest.bookIndex)) books := filteredBooks(books[1].title, interestingWords) } );
+
+output(p);

ecl/regress/serial8b_err.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test that dictionaries are correctly serialized to nonlocal child queries
+// Should get a type incompatibility error on the dataset assignment
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+interestingWords := DICTIONARY([{'elves'},{'cheddar'}], SerialTest.wordRec);
+
+inDs := SerialTest.libraryDatasetFile;
+
+filteredBooks(string searchName, dictionary(SerialTest.wordRec) searchWords) := 
+            SerialTest.bookIndex(
+                KEYED(title = searchName), 
+                EXISTS(words(word IN searchWords))
+            );
+
+
+p := TABLE(inDs, { owner, DATASET(SerialTest.bookDsrec) books := filteredBooks(books[1].title, interestingWords) } );
+
+output(p);

ecl/regress/serial9a.ecl

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test sorting by a dictionary
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+
+x := SORT(SerialTest.libraryDictionaryFile, WHOLE RECORD);
+output(COUNT(NOFOLD(x)));

ecl/regress/serial9b.ecl

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test sorting by a dictionary
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+
+x := SORT(SerialTest.libraryDictionaryFile, WHOLE RECORD, EXCEPT owner);
+output(COUNT(NOFOLD(x)));

ecl/regress/serial9c.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test hashing a dictionary
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+output(TABLE(SerialTest.libraryDictionaryFile, { owner; unsigned8 hashcode := HASH64(books)}));

ecl/regress/serial9d.ecl

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test sorting by a dictionary
+
+#option ('pickBestEngine', false);
+
+IMPORT SerialTest;
+
+
+x := SORT(SerialTest.libraryDictionaryFile, books);
+output(COUNT(NOFOLD(x)));

roxie/ccd/ccdactivities.cpp

@@ -207,7 +207,7 @@ public:
     {
         if (!childQueries.length())
             logctx.setDebuggerActive(false);
-        if (meta.needsDestruct() || meta.needsSerialize() || childQueries.length())
+        if (meta.needsDestruct() || meta.needsSerializeDisk() || childQueries.length())
         {
             Owned<IRoxieSlaveContext> queryContext = queryFactory.createSlaveContext(logctx, packet);
             ForEachItemIn(idx, childQueries)
@@ -323,7 +323,7 @@ protected:
 
     virtual bool needsRowAllocator()
     {
-        return meta.needsSerialize() || meta.isVariableSize();
+        return meta.needsSerializeDisk() || meta.isVariableSize();
     }
 
     virtual void onCreate()
@@ -340,8 +340,8 @@ protected:
             queryContext.setown(basefactory->createChildQueries(basehelper, childGraphs, NULL, logctx, packet));
         if (!queryContext)
             queryContext.setown(basefactory->createSlaveContext(logctx, packet));
-        if (meta.needsSerialize())
-            serializer.setown(meta.createRowSerializer(queryContext->queryCodeContext(), basefactory->queryId()));
+        if (meta.needsSerializeDisk())
+            serializer.setown(meta.createDiskSerializer(queryContext->queryCodeContext(), basefactory->queryId()));
         if (needsRowAllocator())
             rowAllocator.setown(getRowAllocator(meta.queryOriginal(), basefactory->queryId()));
         unsigned parentExtractSize;
@@ -1365,7 +1365,7 @@ public:
     UnkeyedVariableRecordProcessor(IInMemoryIndexCursor *_cursor, CRoxieDiskReadActivity &_owner, IDirectReader *_reader)
       : UnkeyedRecordProcessor(_cursor, _owner, _reader), deserializeSource(_reader)
     {
-        prefetcher.setown(owner.diskSize.queryOriginal()->createRowPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
+        prefetcher.setown(owner.diskSize.queryOriginal()->createDiskPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
     }
 
     virtual void doQuery(IMessagePacker *output, unsigned processed, unsigned __int64 rowLimit, unsigned __int64 stopAfter)
@@ -1821,7 +1821,7 @@ public:
     UnkeyedNormalizeRecordProcessor(IInMemoryIndexCursor *_cursor, CRoxieDiskNormalizeActivity &_owner, IDirectReader *_reader) 
         : NormalizeRecordProcessor(_cursor, _owner), reader(_reader), deserializeSource(_reader)
     {
-        prefetcher.setown(owner.diskSize.queryOriginal()->createRowPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
+        prefetcher.setown(owner.diskSize.queryOriginal()->createDiskPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
     }
 
     virtual void doQuery(IMessagePacker *output, unsigned processed, unsigned __int64 rowLimit, unsigned __int64 stopAfter)
@@ -2100,7 +2100,7 @@ public:
     UnkeyedVariableCountRecordProcessor(IInMemoryIndexCursor *_cursor, CRoxieDiskCountActivity &_owner, IDirectReader *_reader)
         : UnkeyedCountRecordProcessor(_cursor, _owner, _reader), deserializeSource(reader)
     {
-        prefetcher.setown(owner.diskSize.queryOriginal()->createRowPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
+        prefetcher.setown(owner.diskSize.queryOriginal()->createDiskPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
     }
 
     // This version is used for variable size rows 
@@ -2320,10 +2320,10 @@ public:
     {
         helper = (IHThorDiskAggregateArg *) basehelper;
         onCreate();
-        if (meta.needsSerialize())
+        if (meta.needsSerializeDisk())
         {
             // MORE - avoiding serializing to dummy would be more efficient...
-            deserializer.setown(meta.createRowDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
+            deserializer.setown(meta.createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
         }
         CRoxieDiskAggregateActivity *part0 = new CRoxieDiskAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, numParallel, false);
         parts.append(*part0);
@@ -2581,7 +2581,7 @@ public:
     UnkeyedVariableAggregateRecordProcessor(IInMemoryIndexCursor *_cursor, CRoxieDiskAggregateActivity &_owner, IDirectReader *_reader) 
         : UnkeyedAggregateRecordProcessor(_cursor, _owner, _reader), deserializeSource(_reader)
     {
-        prefetcher.setown(owner.diskSize.queryOriginal()->createRowPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
+        prefetcher.setown(owner.diskSize.queryOriginal()->createDiskPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
     }
 
     virtual void doQuery(IMessagePacker *output, unsigned processed, unsigned __int64 rowLimit, unsigned __int64 stopAfter)
@@ -2716,10 +2716,10 @@ public:
     {
         onCreate();
         resultAggregator.start(rowAllocator);
-        if (meta.needsSerialize())
+        if (meta.needsSerializeDisk())
         {
             // MORE - avoiding serializing to dummy would be more efficient...
-            deserializer.setown(meta.createRowDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
+            deserializer.setown(meta.createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
         }
         CRoxieDiskGroupAggregateActivity *part0 = new CRoxieDiskGroupAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, numParallel, false);
         parts.append(*part0);
@@ -2941,7 +2941,7 @@ public:
                                                  ICodeContext *ctx, unsigned activityId)
     : UnkeyedGroupAggregateRecordProcessor(_cursor, _results, _helper, _reader), deserializeSource(_reader)
     {
-        prefetcher.setown(helper.queryDiskRecordSize()->createRowPrefetcher(ctx, activityId));
+        prefetcher.setown(helper.queryDiskRecordSize()->createDiskPrefetcher(ctx, activityId));
     }
 
     virtual void doQuery(IMessagePacker *output, unsigned processed, unsigned __int64 rowLimit, unsigned __int64 stopAfter)
@@ -4355,7 +4355,7 @@ public:
         IHThorFetchContext * fetchContext = static_cast<IHThorFetchContext *>(helper->selectInterface(TAIfetchcontext_1));
         IOutputMetaData *diskMeta = fetchContext->queryDiskRecordSize();
         diskAllocator.setown(getRowAllocator(diskMeta, basefactory->queryId()));
-        rowDeserializer.setown(diskMeta->createRowDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
+        rowDeserializer.setown(diskMeta->createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
     }
 
     virtual size32_t doFetch(ARowBuilder & rowBuilder, offset_t pos, offset_t rawpos, void *inputData)
@@ -4995,7 +4995,7 @@ bool CRoxieKeyedJoinFetchActivity::process()
     unsigned skipped = 0;
     unsigned __int64 rowLimit = helper->getRowLimit();
     unsigned totalSizeSent = 0;
-    Owned<IOutputRowDeserializer> rowDeserializer = helper->queryDiskRecordSize()->createRowDeserializer(queryContext->queryCodeContext(), basefactory->queryId());
+    Owned<IOutputRowDeserializer> rowDeserializer = helper->queryDiskRecordSize()->createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId());
     Owned<IEngineRowAllocator> diskAllocator = getRowAllocator(helper->queryDiskRecordSize(), basefactory->queryId());
     RtlDynamicRowBuilder diskRowBuilder(diskAllocator);
 

roxie/ccd/ccdserver.cpp

@@ -3659,9 +3659,9 @@ public:
         ctx = _ctx;
         debugContext = ctx->queryDebugContext();
         colocalArg = _colocalArg;
-        if (meta.needsSerialize())
+        if (meta.needsSerializeDisk())
         {
-            deserializer.setown(meta.createRowDeserializer(_ctx->queryCodeContext(), activity.queryId()));
+            deserializer.setown(meta.createDiskDeserializer(_ctx->queryCodeContext(), activity.queryId()));
             rowAllocator.setown(ctx->queryCodeContext()->getRowAllocator(meta.queryOriginal(), activity.queryId()));
         }
         if (ctx->queryDebugContext() && ctx->queryDebugContext()->getExecuteSequentially())
@@ -5015,7 +5015,7 @@ public:
     virtual void onCreate(IRoxieSlaveContext *_ctx, IHThorArg *_colocalParent)
     {
         CRoxieServerActivity::onCreate(_ctx, _colocalParent);
-        rowDeserializer.setown(rowAllocator->createRowDeserializer(ctx->queryCodeContext()));
+        rowDeserializer.setown(rowAllocator->createDiskDeserializer(ctx->queryCodeContext()));
     }
 
     virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
@@ -8715,7 +8715,7 @@ public:
     virtual void onCreate(IRoxieSlaveContext *_ctx, IHThorArg *_colocalParent)
     {
         CRoxieServerActivity::onCreate(_ctx, _colocalParent);
-        rowDeserializer.setown(rowAllocator->createRowDeserializer(ctx->queryCodeContext()));
+        rowDeserializer.setown(rowAllocator->createDiskDeserializer(ctx->queryCodeContext()));
     }
 
     virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
@@ -8826,8 +8826,8 @@ public:
     virtual void onCreate(IRoxieSlaveContext *_ctx, IHThorArg *_colocalParent)
     {
         CRoxieServerActivity::onCreate(_ctx, _colocalParent);
-        rowSerializer.setown(inputMeta.createRowSerializer(ctx->queryCodeContext(), activityId));
-        rowDeserializer.setown(rowAllocator->createRowDeserializer(ctx->queryCodeContext()));
+        rowSerializer.setown(inputMeta.createDiskSerializer(ctx->queryCodeContext(), activityId));
+        rowDeserializer.setown(rowAllocator->createDiskDeserializer(ctx->queryCodeContext()));
         writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
     }
 
@@ -9017,7 +9017,7 @@ public:
     {
         CRoxieServerActivity::onCreate(_ctx, _colocalParent);
         inputMeta.set(input->queryOutputMeta());
-        rowSerializer.setown(inputMeta.createRowSerializer(ctx->queryCodeContext(), activityId));
+        rowSerializer.setown(inputMeta.createDiskSerializer(ctx->queryCodeContext(), activityId));
         writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
     }
 
@@ -14409,10 +14409,12 @@ public:
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
     virtual void destruct(byte * self)  {}
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputMetaData * querySerializedMeta() { return this; }
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
 };
 
@@ -19265,7 +19267,7 @@ public:
         if (serverContext->outputResultsToWorkUnit()||(response && response->isRaw))
         {
             createRowAllocator();
-            rowSerializer.setown(rowAllocator->createRowSerializer(ctx->queryCodeContext()));
+            rowSerializer.setown(rowAllocator->createDiskSerializer(ctx->queryCodeContext()));
         }
         __int64 initialProcessed = processed;
         RtlLinkedDatasetBuilder builder(rowAllocator);
@@ -19744,7 +19746,7 @@ public:
                 {
                     reader.setown(manager->createReader(0, 0, 1));
                     deserializeSource.setStream(reader);
-                    prefetcher.setown(diskSize.queryOriginal()->createRowPrefetcher(ctx->queryCodeContext(), activityId));
+                    prefetcher.setown(diskSize.queryOriginal()->createDiskPrefetcher(ctx->queryCodeContext(), activityId));
                 }
                 helper.setCallback(reader ? reader->queryThorDiskCallback() : cursor);
             }
@@ -28526,7 +28528,7 @@ public:
         size32_t count = counts.item(id);
 
         MemoryBuffer result;
-        Owned<IOutputRowSerializer> rowSerializer = meta->createRowSerializer(codectx, 0); // NOTE - we don't have a meaningful activity id. Only used for error reporting.
+        Owned<IOutputRowSerializer> rowSerializer = meta->createDiskSerializer(codectx, 0); // NOTE - we don't have a meaningful activity id. Only used for error reporting.
         bool grouped = meta->isGrouped();
         for (size32_t idx = 0; idx<count; idx++)
         {
@@ -29855,7 +29857,7 @@ public:
             eof = false;
             eogPending = false;
             bufferBase = NULL;
-            rowDeserializer.setown(rowAllocator->createRowDeserializer(parent->queryCodeContext()));
+            rowDeserializer.setown(rowAllocator->createDiskDeserializer(parent->queryCodeContext()));
             bufferStream.setown(createMemoryBufferSerialStream(blockBuffer));
             rowSource.setStream(bufferStream);
         }
@@ -32263,10 +32265,12 @@ public:
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
     virtual void destruct(byte * self)  {}
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
-    virtual IOutputMetaData * querySerializedMeta() { return NULL; }
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputMetaData * querySerializedDiskMeta() { return NULL; }
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
 } testMeta;
 

roxie/roxiemem/roxierow.cpp

@@ -181,13 +181,21 @@ public:
     {
         return idStr.append(activityId); // MORE - may want more context info in here
     }
-    virtual IOutputRowSerializer *createRowSerializer(ICodeContext *ctx)
+    virtual IOutputRowSerializer *createDiskSerializer(ICodeContext *ctx)
     {
-        return meta.createRowSerializer(ctx, activityId);
+        return meta.createDiskSerializer(ctx, activityId);
     }
-    virtual IOutputRowDeserializer *createRowDeserializer(ICodeContext *ctx)
+    virtual IOutputRowDeserializer *createDiskDeserializer(ICodeContext *ctx)
     {
-        return meta.createRowDeserializer(ctx, activityId);
+        return meta.createDiskDeserializer(ctx, activityId);
+    }
+    virtual IOutputRowSerializer *createInternalSerializer(ICodeContext *ctx)
+    {
+        return meta.createInternalSerializer(ctx, activityId);
+    }
+    virtual IOutputRowDeserializer *createInternalDeserializer(ICodeContext *ctx)
+    {
+        return meta.createInternalDeserializer(ctx, activityId);
     }
 
 protected:
@@ -555,12 +563,14 @@ protected:
         virtual void toXML(const byte * self, IXmlWriter & out) {}
         virtual unsigned getVersion() const { return 0; }
         virtual unsigned getMetaFlags() { return 0; }
-        virtual IOutputMetaData * querySerializedMeta() { return this; }
+        virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
 
         virtual void destruct(byte * self) {}
-        virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-        virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
-        virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
+        virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+        virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+        virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) { return NULL; }
+        virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+        virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
         virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
 
         size32_t minSize;

rtl/eclrtl/eclrtl.hpp

@@ -537,6 +537,7 @@ ECLRTL_API UChar * deserializeVUnicodeX(MemoryBuffer &in);
 ECLRTL_API void deserializeQStrX(size32_t & len, char * & data, MemoryBuffer &out);
 ECLRTL_API void deserializeRowsetX(size32_t & count, byte * * & data, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in);
 ECLRTL_API void deserializeGroupedRowsetX(size32_t & count, byte * * & data, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in);
+ECLRTL_API void deserializeDictionaryX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in);
 
 ECLRTL_API byte * rtlDeserializeRow(IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, const void * src);
 ECLRTL_API byte * rtlDeserializeBufferRow(IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer & buffer);
@@ -552,6 +553,7 @@ ECLRTL_API void serializeQStrX(size32_t len, const char * data, MemoryBuffer &ou
 ECLRTL_API void serializeRowsetX(size32_t count, byte * * data, IOutputRowSerializer * serializer, MemoryBuffer &out);
 ECLRTL_API void serializeGroupedRowsetX(size32_t count, byte * * data, IOutputRowSerializer * serializer, MemoryBuffer &out);
 ECLRTL_API void serializeRow(const void * row, IOutputRowSerializer * serializer, MemoryBuffer & out);
+ECLRTL_API void serializeDictionaryX(size32_t count, byte * * rows, IOutputRowSerializer * serializer, MemoryBuffer & buffer);
 
 ECLRTL_API void serializeFixedString(unsigned len, const char *field, MemoryBuffer &out);
 ECLRTL_API void serializeLPString(unsigned len, const char *field, MemoryBuffer &out);

rtl/eclrtl/rtlds.cpp

@@ -419,7 +419,7 @@ void RtlLinkedDatasetBuilder::cloneRow(size32_t len, const void * row)
     memcpy(self, row, len);
     
     IOutputMetaData * meta = rowAllocator->queryOutputMeta();
-    if (meta->getMetaFlags() & MDFneedserialize)
+    if (meta->getMetaFlags() & MDFneeddestruct)
     {
         RtlChildRowLinkerWalker walker;
         meta->walkIndirectMembers(self, walker);
@@ -437,48 +437,6 @@ void RtlLinkedDatasetBuilder::deserializeRow(IOutputRowDeserializer & deserializ
 }
 
 
-inline void doDeserializeRowset(RtlLinkedDatasetBuilder & builder, IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, offset_t marker, bool isGrouped)
-{
-    byte eogPending = false;
-    while (!in.finishedNested(marker))
-    {
-        if (isGrouped && eogPending)
-            builder.appendEOG();
-        builder.deserializeRow(deserializer, in);
-        if (isGrouped)
-            in.read(1, &eogPending);
-    }
-}
-
-inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows, bool isGrouped)
-{
-    for (unsigned i=0; i < count; i++)
-    {
-        byte *row = rows[i];
-        if (row)
-        {
-            serializer->serialize(out, rows[i]);
-            if (isGrouped)
-            {
-                byte eogPending = (i+1 < count) && (rows[i+1] == NULL);
-                out.put(1, &eogPending);
-            }
-        }
-        else
-        {
-            assert(isGrouped); // should not be seeing NULLs otherwise - should not use this function for DICTIONARY
-        }
-    }
-}
-
-
-void RtlLinkedDatasetBuilder::deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped)
-{
-    offset_t marker = in.beginNested();
-    doDeserializeRowset(*this, deserializer, in, marker, isGrouped);
-}
-
-
 void RtlLinkedDatasetBuilder::finalizeRows()
 {
     if (count != max)
@@ -542,43 +500,6 @@ void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngine
 }
 
 
-inline void doDeserializeRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in, bool isGrouped)
-{
-    RtlLinkedDatasetBuilder builder(_rowAllocator);
-
-    builder.deserialize(*deserializer, in, isGrouped);
-
-    count = builder.getcount();
-    rowset = builder.linkrows();
-}
-
-
-extern ECLRTL_API void rtlDeserializeRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in)
-{
-    doDeserializeRowset(count, rowset, _rowAllocator, deserializer, in, false);
-}
-
-
-extern ECLRTL_API void rtlDeserializeGroupedRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in)
-{
-    doDeserializeRowset(count, rowset, _rowAllocator, deserializer, in, true);
-}
-
-
-void rtlSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
-{
-    size32_t marker = out.beginNested();
-    doSerializeRowset(out, serializer, count, rows, false);
-    out.endNested(marker);
-}
-
-void rtlSerializeGroupedRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
-{
-    size32_t marker = out.beginNested();
-    doSerializeRowset(out, serializer, count, rows, true);
-    out.endNested(marker);
-}
-
 //---------------------------------------------------------------------------
 
 RtlLinkedDictionaryBuilder::RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialSize)
@@ -678,6 +599,13 @@ void RtlLinkedDictionaryBuilder::checkSpace()
     }
 }
 
+void RtlLinkedDictionaryBuilder::deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in)
+{
+    builder.ensureRow();
+    size32_t rowSize = deserializer.deserialize(builder, in);
+    finalizeRow(rowSize);
+}
+
 void RtlLinkedDictionaryBuilder::appendRows(size32_t num, byte * * rows)
 {
     // MORE - if we know that the source is already a hashtable, we can optimize the add to an empty table...
@@ -698,7 +626,7 @@ void RtlLinkedDictionaryBuilder::cloneRow(size32_t len, const void * row)
     memcpy(self, row, len);
 
     IOutputMetaData * meta = rowAllocator->queryOutputMeta();
-    if (meta->getMetaFlags() & MDFneedserialize)
+    if (meta->getMetaFlags() & MDFneeddestruct)
     {
         RtlChildRowLinkerWalker walker;
         meta->walkIndirectMembers(self, walker);
@@ -758,66 +686,16 @@ extern ECLRTL_API bool rtlDictionaryLookupExists(IHThorHashLookupInfo &hashInfo,
     }
 }
 
-extern ECLRTL_API void rtlSerializeDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
-{
-    out.put(sizeof(count), &count);
-    size32_t idx = 0;
-    while (idx < count)
-    {
-        byte numRows = 0;
-        while (numRows < 255 && idx+numRows < count && rows[idx+numRows] != NULL)
-            numRows++;
-        out.put(1, &numRows);
-        for (int i = 0; i < numRows; i++)
-        {
-            byte *nextrec = rows[idx+i];
-            assert(nextrec);
-            serializer->serialize(out, nextrec);
-        }
-        idx += numRows;
-        byte numNulls = 0;
-        while (numNulls < 255 && idx+numNulls < count && rows[idx+numNulls] == NULL)
-            numNulls++;
-        out.put(1, &numNulls);
-        idx += numNulls;
-    }
-}
-
-extern ECLRTL_API void rtlDictionary2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src)
-{
-    RtlLinkedDatasetBuilder builder(rowAllocator);
-    Owned<ISerialStream> stream = createMemorySerialStream(src, lenSrc);
-    CThorStreamDeserializerSource source(stream);
-
-    size32_t totalRows;
-    source.read(sizeof(totalRows), &totalRows);
-    builder.ensure(totalRows);
-    byte nullsPending = 0;
-    byte rowsPending = 0;
-    while (!source.finishedNested(lenSrc))
-    {
-        source.read(1, &rowsPending);
-        for (int i = 0; i < rowsPending; i++)
-            builder.deserializeRow(*deserializer, source);
-        source.read(1, &nullsPending);
-        for (int i = 0; i < nullsPending; i++)
-            builder.append(NULL);
-    }
-
-    count = builder.getcount();
-    assertex(count==totalRows);
-    rowset = builder.linkrows();
-}
-
-//---------------------------------------------------------------------------
+//---------------------------------------------------------------------------------------------------------------------
+// Serialization helper classes
 
 //These definitions should be shared with thorcommon, but to do that
 //they would need to be moved to an rtlds.ipp header, which thorcommon then included.
 
-class ECLRTL_API CThorRtlRowSerializer : implements IRowSerializerTarget
+class ECLRTL_API CMemoryBufferSerializeTarget : implements IRowSerializerTarget
 {
 public:
-    CThorRtlRowSerializer(MemoryBuffer & _buffer) : buffer(_buffer)
+    CMemoryBufferSerializeTarget(MemoryBuffer & _buffer) : buffer(_buffer)
     {
     }
 
@@ -845,102 +723,227 @@ protected:
     MemoryBuffer & buffer;
 };
 
+class ECLRTL_API CRowBuilderSerializeTarget : implements IRowSerializerTarget
+{
+public:
+    CRowBuilderSerializeTarget(ARowBuilder & _builder) : builder(_builder)
+    {
+        offset = 0;
+    }
 
-inline void doDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src, bool isGrouped)
+    virtual void put(size32_t len, const void * ptr)
+    {
+        byte * data = builder.ensureCapacity(offset + len, "");
+        memcpy(data+offset, ptr, len);
+        offset += len;
+    }
+
+    virtual size32_t beginNested()
+    {
+        unsigned pos = offset;
+        offset += sizeof(size32_t);
+        return pos;
+    }
+
+    virtual void endNested(size32_t sizePos)
+    {
+        byte * self = builder.getSelf();
+        *(size32_t *)(self + sizePos) = offset - (sizePos + sizeof(size32_t));
+    }
+
+    inline size32_t length() const { return offset; }
+
+protected:
+    ARowBuilder & builder;
+    size32_t offset;
+};
+
+//---------------------------------------------------------------------------------------------------------------------
+// internal serialization helper functions
+
+
+inline void doDeserializeRowset(RtlLinkedDatasetBuilder & builder, IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, offset_t marker, bool isGrouped)
 {
-    RtlLinkedDatasetBuilder builder(rowAllocator);
-    Owned<ISerialStream> stream = createMemorySerialStream(src, lenSrc);
-    CThorStreamDeserializerSource source(stream);
+    byte eogPending = false;
+    while (!in.finishedNested(marker))
+    {
+        if (isGrouped && eogPending)
+            builder.appendEOG();
+        builder.deserializeRow(deserializer, in);
+        if (isGrouped)
+            in.read(1, &eogPending);
+    }
+}
+
+inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows, bool isGrouped)
+{
+    for (unsigned i=0; i < count; i++)
+    {
+        byte *row = rows[i];
+        if (row)
+        {
+            serializer->serialize(out, rows[i]);
+            if (isGrouped)
+            {
+                byte eogPending = (i+1 < count) && (rows[i+1] == NULL);
+                out.put(1, &eogPending);
+            }
+        }
+        else
+        {
+            assert(isGrouped); // should not be seeing NULLs otherwise - should not use this function for DICTIONARY
+        }
+    }
+}
+
+
+inline void doSerializeRowsetStripNulls(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    for (unsigned i=0; i < count; i++)
+    {
+        byte *row = rows[i];
+        if (row)
+            serializer->serialize(out, rows[i]);
+    }
+}
+
+inline void doDeserializeRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, offset_t marker, IRowDeserializerSource & in, bool isGrouped)
+{
+    RtlLinkedDatasetBuilder builder(_rowAllocator);
 
-    doDeserializeRowset(builder, *deserializer, source, lenSrc, isGrouped);
+    doDeserializeRowset(builder, *deserializer, in, marker, isGrouped);
 
     count = builder.getcount();
     rowset = builder.linkrows();
 }
 
-inline void doRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows, bool isGrouped)
+
+inline void doDeserializeChildRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in, bool isGrouped)
 {
-    MemoryBuffer buffer;
-    CThorRtlRowSerializer out(buffer);
+    offset_t marker = in.beginNested();
+    doDeserializeRowset(count, rowset, _rowAllocator, deserializer, marker, in, isGrouped);
+}
 
-    doSerializeRowset(out, serializer, count, rows, isGrouped);
 
-    rtlFree(tgt);
-    tlen = buffer.length();
-    tgt = buffer.detach();      // not strictly speaking correct - it should have been allocated with rtlMalloc();
+//--------------------------------------------------------------------------------------------------------------------
+//Serialize/deserialize functions call for child datasets in the row serializer
+
+extern ECLRTL_API void rtlDeserializeChildRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in)
+{
+    doDeserializeChildRowset(count, rowset, _rowAllocator, deserializer, in, false);
 }
 
-extern ECLRTL_API void rtlDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src, bool isGrouped)
+
+extern ECLRTL_API void rtlDeserializeChildGroupedRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in)
 {
-    doDataset2RowsetX(count, rowset, rowAllocator, deserializer, lenSrc, src, isGrouped);
+    doDeserializeChildRowset(count, rowset, _rowAllocator, deserializer, in, true);
 }
 
-extern ECLRTL_API void rtlDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src)
+
+void rtlSerializeChildRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
 {
-    doDataset2RowsetX(count, rowset, rowAllocator, deserializer, lenSrc, src, false);
+    size32_t marker = out.beginNested();
+    doSerializeRowset(out, serializer, count, rows, false);
+    out.endNested(marker);
 }
 
-extern ECLRTL_API void rtlGroupedDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src)
+void rtlSerializeChildGroupedRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
 {
-    doDataset2RowsetX(count, rowset, rowAllocator, deserializer, lenSrc, src, true);
+    size32_t marker = out.beginNested();
+    doSerializeRowset(out, serializer, count, rows, true);
+    out.endNested(marker);
 }
 
-extern ECLRTL_API void rtlRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows, bool isGrouped)
+//--------------------------------------------------------------------------------------------------------------------
+//Serialize/deserialize functions used to serialize data from the master to the slave (defined in eclrtl.hpp)  to/from a MemoryBuffer
+
+extern void deserializeRowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in)
 {
-    doRowset2DatasetX(tlen, tgt, serializer, count, rows, isGrouped);
+    Owned<ISerialStream> stream = createMemoryBufferSerialStream(in);
+    CThorStreamDeserializerSource rowSource(stream);
+    doDeserializeChildRowset(count, rowset, _rowAllocator, deserializer, rowSource, false);
 }
 
-extern ECLRTL_API void rtlRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+extern void deserializeGroupedRowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in)
 {
-    doRowset2DatasetX(tlen, tgt, serializer, count, rows, false);
+    Owned<ISerialStream> stream = createMemoryBufferSerialStream(in);
+    CThorStreamDeserializerSource rowSource(stream);
+    doDeserializeChildRowset(count, rowset, _rowAllocator, deserializer, rowSource, true);
 }
 
-extern ECLRTL_API void rtlGroupedRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+extern void serializeRowsetX(size32_t count, byte * * rows, IOutputRowSerializer * serializer, MemoryBuffer & buffer)
 {
-    doRowset2DatasetX(tlen, tgt, serializer, count, rows, true);
+    CMemoryBufferSerializeTarget out(buffer);
+    rtlSerializeChildRowset(out, serializer, count, rows);
 }
 
-extern ECLRTL_API void rtlRowset2DictionaryX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+extern void serializeGroupedRowsetX(size32_t count, byte * * rows, IOutputRowSerializer * serializer, MemoryBuffer & buffer)
 {
-    MemoryBuffer rowdata;
-    CThorRtlRowSerializer out(rowdata);
-    rtlSerializeDictionary(out, serializer, count, rows);
+    CMemoryBufferSerializeTarget out(buffer);
+    rtlSerializeChildGroupedRowset(out, serializer, count, rows);
+}
 
-    rtlFree(tgt);
-    tlen = rowdata.length();
-    tgt = rowdata.detach();
+
+//--------------------------------------------------------------------------------------------------------------------
+// Functions for converting between different representations - where the source/target are complete datasets
+
+inline void doDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src, bool isGrouped)
+{
+    Owned<ISerialStream> stream = createMemorySerialStream(src, lenSrc);
+    CThorStreamDeserializerSource source(stream);
+
+    doDeserializeRowset(count, rowset, rowAllocator, deserializer, lenSrc, source, isGrouped);
 }
 
-void deserializeRowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in)
+extern ECLRTL_API void rtlDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src, bool isGrouped)
 {
-    Owned<ISerialStream> stream = createMemoryBufferSerialStream(in);
-    CThorStreamDeserializerSource rowSource(stream);
-    doDeserializeRowset(count, rowset, _rowAllocator, deserializer, rowSource, false);
+    doDataset2RowsetX(count, rowset, rowAllocator, deserializer, lenSrc, src, isGrouped);
 }
 
-void deserializeGroupedRowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in)
+extern ECLRTL_API void rtlDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src)
 {
-    Owned<ISerialStream> stream = createMemoryBufferSerialStream(in);
-    CThorStreamDeserializerSource rowSource(stream);
-    doDeserializeRowset(count, rowset, _rowAllocator, deserializer, rowSource, true);
+    doDataset2RowsetX(count, rowset, rowAllocator, deserializer, lenSrc, src, false);
+}
+
+extern ECLRTL_API void rtlGroupedDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src)
+{
+    doDataset2RowsetX(count, rowset, rowAllocator, deserializer, lenSrc, src, true);
 }
 
-void serializeRowsetX(size32_t count, byte * * rows, IOutputRowSerializer * serializer, MemoryBuffer & buffer)
+
+inline void doRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows, bool isGrouped)
+{
+    MemoryBuffer buffer;
+    CMemoryBufferSerializeTarget out(buffer);
+
+    doSerializeRowset(out, serializer, count, rows, isGrouped);
+
+    rtlFree(tgt);
+    tlen = buffer.length();
+    tgt = buffer.detach();      // not strictly speaking correct - it should have been allocated with rtlMalloc();
+}
+
+extern ECLRTL_API void rtlRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows, bool isGrouped)
 {
-    CThorRtlRowSerializer out(buffer);
-    rtlSerializeRowset(out, serializer, count, rows);
+    doRowset2DatasetX(tlen, tgt, serializer, count, rows, isGrouped);
 }
 
-void serializeGroupedRowsetX(size32_t count, byte * * rows, IOutputRowSerializer * serializer, MemoryBuffer & buffer)
+extern ECLRTL_API void rtlRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
 {
-    CThorRtlRowSerializer out(buffer);
-    rtlSerializeGroupedRowset(out, serializer, count, rows);
+    doRowset2DatasetX(tlen, tgt, serializer, count, rows, false);
 }
 
+extern ECLRTL_API void rtlGroupedRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    doRowset2DatasetX(tlen, tgt, serializer, count, rows, true);
+}
+
+//--------------------------------------------------------------------------------------------------------------------
+// Serialize/deserialize rows to a memory buffer
 
 void serializeRow(const void * row, IOutputRowSerializer * serializer, MemoryBuffer & buffer)
 {
-    CThorRtlRowSerializer out(buffer);
+    CMemoryBufferSerializeTarget out(buffer);
     serializer->serialize(out, static_cast<const byte *>(row));
 }
 
@@ -955,6 +958,8 @@ extern ECLRTL_API byte * rtlDeserializeBufferRow(IEngineRowAllocator * rowAlloca
     return static_cast<byte *>(const_cast<void *>(rowBuilder.finalizeRowClear(rowSize)));
 }
 
+//--------------------------------------------------------------------------------------------------------------------
+// serialize/deserialize a row to a builder or another row
 
 extern ECLRTL_API byte * rtlDeserializeRow(IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, const void * src)
 {
@@ -967,65 +972,172 @@ extern ECLRTL_API byte * rtlDeserializeRow(IEngineRowAllocator * rowAllocator, I
 }
 
 
-extern ECLRTL_API size32_t rtlSerializeRow(size32_t lenOut, void * out, IOutputRowSerializer * serializer, const void * src)
+extern ECLRTL_API size32_t rtlDeserializeToBuilder(ARowBuilder & builder, IOutputRowDeserializer * deserializer, const void * src)
 {
-    MemoryBuffer buffer;
-    CThorRtlRowSerializer result(buffer);
-    buffer.setBuffer(lenOut, out, false);
-    buffer.setWritePos(0);
-    serializer->serialize(result, (const byte *)src);
-    return buffer.length();
+    Owned<ISerialStream> stream = createMemorySerialStream(src, 0x7fffffff);
+    CThorStreamDeserializerSource source(stream);
+    return deserializer->deserialize(builder, source);
 }
 
+extern ECLRTL_API size32_t rtlSerializeToBuilder(ARowBuilder & builder, IOutputRowSerializer * serializer, const void * src)
+{
+    CRowBuilderSerializeTarget target(builder);
+    serializer->serialize(target, (const byte *)src);
+    return target.length();
+}
+
+//--------------------------------------------------------------------------------------------------------------------
 
-class ECLRTL_API CThorRtlBuilderSerializer : implements IRowSerializerTarget
+static void doSerializeDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
 {
-public:
-    CThorRtlBuilderSerializer(ARowBuilder & _builder) : builder(_builder)
+    out.put(sizeof(count), &count);
+    size32_t idx = 0;
+    while (idx < count)
     {
-        offset = 0;
+        byte numRows = 0;
+        while (numRows < 255 && idx+numRows < count && rows[idx+numRows] != NULL)
+            numRows++;
+        out.put(1, &numRows);
+        for (int i = 0; i < numRows; i++)
+        {
+            byte *nextrec = rows[idx+i];
+            assert(nextrec);
+            serializer->serialize(out, nextrec);
+        }
+        idx += numRows;
+        byte numNulls = 0;
+        while (numNulls < 255 && idx+numNulls < count && rows[idx+numNulls] == NULL)
+            numNulls++;
+        out.put(1, &numNulls);
+        idx += numNulls;
     }
+}
 
-    virtual void put(size32_t len, const void * ptr)
-    {
-        byte * data = builder.ensureCapacity(offset + len, "");
-        memcpy(data+offset, ptr, len);
-        offset += len;
-    }
+static void doDeserializeDictionary(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, offset_t marker, IRowDeserializerSource & in)
+{
+    RtlLinkedDatasetBuilder builder(rowAllocator);
 
-    virtual size32_t beginNested()
+    size32_t totalRows;
+    in.read(sizeof(totalRows), &totalRows);
+    builder.ensure(totalRows);
+    byte nullsPending = 0;
+    byte rowsPending = 0;
+    while (!in.finishedNested(marker))
     {
-        unsigned pos = offset;
-        offset += sizeof(size32_t);
-        return pos;
+        in.read(1, &rowsPending);
+        for (int i = 0; i < rowsPending; i++)
+            builder.deserializeRow(*deserializer, in);
+        in.read(1, &nullsPending);
+        for (int i = 0; i < nullsPending; i++)
+            builder.appendEOG();
     }
 
-    virtual void endNested(size32_t sizePos)
-    {
-        byte * self = builder.getSelf();
-        *(size32_t *)(self + sizePos) = offset - (sizePos + sizeof(size32_t));
-    }
+    count = builder.getcount();
+    assertex(count==totalRows);
+    rowset = builder.linkrows();
+}
 
-    inline size32_t length() const { return offset; }
 
-protected:
-    ARowBuilder & builder;
-    size32_t offset;
-};
+static void doDeserializeDictionaryFromDataset(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, IHThorHashLookupInfo & hashInfo, offset_t marker, IRowDeserializerSource & in)
+{
+    RtlLinkedDictionaryBuilder builder(rowAllocator, &hashInfo);
 
+    while (!in.finishedNested(marker))
+        builder.deserializeRow(*deserializer, in);
 
-extern ECLRTL_API size32_t rtlDeserializeToBuilder(ARowBuilder & builder, IOutputRowDeserializer * deserializer, const void * src)
+    count = builder.getcount();
+    rowset = builder.linkrows();
+}
+
+extern ECLRTL_API void rtlSerializeDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
 {
-    Owned<ISerialStream> stream = createMemorySerialStream(src, 0x7fffffff);
-    CThorStreamDeserializerSource source(stream);
-    return deserializer->deserialize(builder, source);
+    doSerializeDictionary(out, serializer, count, rows);
 }
 
-extern ECLRTL_API size32_t rtlSerializeToBuilder(ARowBuilder & builder, IOutputRowSerializer * serializer, const void * src)
+extern ECLRTL_API void rtlSerializeDictionaryToDataset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
 {
-    CThorRtlBuilderSerializer target(builder);
-    serializer->serialize(target, (const byte *)src);
-    return target.length();
+    doSerializeRowsetStripNulls(out, serializer, count, rows);
+}
+
+extern ECLRTL_API void rtlDeserializeChildDictionary(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in)
+{
+    offset_t marker = in.beginNested(); // MORE: Would this be better as a count?
+    doDeserializeDictionary(count, rowset, rowAllocator, deserializer, marker, in);
+}
+
+extern ECLRTL_API void rtlDeserializeChildDictionaryFromDataset(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, IHThorHashLookupInfo & hashInfo, IRowDeserializerSource & in)
+{
+    offset_t marker = in.beginNested(); // MORE: Would this be better as a count?
+    doDeserializeDictionaryFromDataset(count, rowset, rowAllocator, deserializer, hashInfo, marker, in);
+}
+
+extern ECLRTL_API void rtlSerializeChildDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    size32_t marker = out.beginNested();
+    doSerializeDictionary(out, serializer, count, rows);
+    out.endNested(marker);
+}
+
+extern ECLRTL_API void rtlSerializeChildDictionaryToDataset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    size32_t marker = out.beginNested();
+    doSerializeRowsetStripNulls(out, serializer, count, rows);
+    out.endNested(marker);
+}
+
+extern void deserializeDictionaryX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in)
+{
+    Owned<ISerialStream> stream = createMemoryBufferSerialStream(in);
+    CThorStreamDeserializerSource rowSource(stream);
+    rtlDeserializeChildDictionary(count, rowset, _rowAllocator, deserializer, rowSource);
+}
+
+extern void serializeDictionaryX(size32_t count, byte * * rows, IOutputRowSerializer * serializer, MemoryBuffer & buffer)
+{
+    CMemoryBufferSerializeTarget out(buffer);
+    rtlSerializeChildDictionary(out, serializer, count, rows);
+}
+
+
+extern ECLRTL_API void rtlDeserializeDictionary(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src)
+{
+    Owned<ISerialStream> stream = createMemorySerialStream(src, lenSrc);
+    CThorStreamDeserializerSource in(stream);
+
+    doDeserializeDictionary(count, rowset, rowAllocator, deserializer, lenSrc, in);
+}
+
+
+extern ECLRTL_API void rtlDeserializeDictionaryFromDataset(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, IHThorHashLookupInfo & hashInfo, size32_t lenSrc, const void * src)
+{
+    Owned<ISerialStream> stream = createMemorySerialStream(src, lenSrc);
+    CThorStreamDeserializerSource in(stream);
+
+    doDeserializeDictionaryFromDataset(count, rowset, rowAllocator, deserializer, hashInfo, lenSrc, in);
+}
+
+extern ECLRTL_API void rtlSerializeDictionary(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    MemoryBuffer buffer;
+    CMemoryBufferSerializeTarget out(buffer);
+
+    doSerializeDictionary(out, serializer, count, rows);
+
+    rtlFree(tgt);
+    tlen = buffer.length();
+    tgt = buffer.detach();      // not strictly speaking correct - it should have been allocated with rtlMalloc();
+}
+
+extern ECLRTL_API void rtlSerializeDictionaryToDataset(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    MemoryBuffer buffer;
+    CMemoryBufferSerializeTarget out(buffer);
+
+    doSerializeRowsetStripNulls(out, serializer, count, rows);
+
+    rtlFree(tgt);
+    tlen = buffer.length();
+    tgt = buffer.detach();      // not strictly speaking correct - it should have been allocated with rtlMalloc();
 }
 
 //---------------------------------------------------------------------------

rtl/eclrtl/rtlds_imp.hpp

@@ -332,7 +332,7 @@ class ECLRTL_API CSimpleSourceRowPrefetcher : public ISourceRowPrefetcher, publi
 public:
     CSimpleSourceRowPrefetcher(IOutputMetaData & _meta, ICodeContext * _ctx, unsigned _activityId)
     {
-        deserializer.setown(_meta.querySerializedMeta()->createRowDeserializer(_ctx, _activityId));
+        deserializer.setown(_meta.querySerializedDiskMeta()->createDiskDeserializer(_ctx, _activityId));
         rowAllocator.setown(_ctx->getRowAllocator(&_meta, _activityId));
     }
 
@@ -363,7 +363,6 @@ public:
     inline void appendEOG() { appendOwn(NULL); }
     byte * createRow();
     void cloneRow(size32_t len, const void * ptr);
-    void deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped);
     void deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in);
     void expand(size32_t required);
     void resize(size32_t required);
@@ -407,10 +406,10 @@ public:
     void finalizeRow(size32_t len);
     inline RtlDynamicRowBuilder & rowBuilder() { return builder; }
     void cloneRow(size32_t len, const void * ptr);
+    void deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in);
+
     /*
         Not clear which if any of these we will want...
-    void deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped);
-    void deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in);
     void expand(size32_t required);
     void resize(size32_t required);
     void finalizeRows();
@@ -443,11 +442,15 @@ extern ECLRTL_API bool rtlDictionaryLookupExists(IHThorHashLookupInfo &hashInfo,
 extern ECLRTL_API void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngineRowAllocator * rowAllocator, size32_t count, byte * * rows);
 
 
-extern ECLRTL_API void rtlDeserializeRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in);
-extern ECLRTL_API void rtlDeserializeRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src);
-extern ECLRTL_API void rtlSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
-extern ECLRTL_API void rtlSerializeRowset(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+//Functions that are used to convert between the serialized and internal memory format
+//functions containing child also serialize the length of the dataset, functions without it pass it independently
 
+extern ECLRTL_API void rtlDeserializeChildRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in);
+extern ECLRTL_API void rtlDeserializeChildGroupRowset(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in);
+extern ECLRTL_API void rtlSerializeChildRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+extern ECLRTL_API void rtlSerializeChildGroupRowset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+
+//Functions for converting between rowsets and complete datasets (where length is known)
 extern ECLRTL_API void rtlDataset2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src, bool isGrouped);
 extern ECLRTL_API void rtlRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows, bool isGrouped);
 
@@ -456,15 +459,24 @@ extern ECLRTL_API void rtlGroupedDataset2RowsetX(size32_t & count, byte * * & ro
 extern ECLRTL_API void rtlRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
 extern ECLRTL_API void rtlGroupedRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
 
-extern ECLRTL_API void rtlSerializeDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
-extern ECLRTL_API void rtlDictionary2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src);
-extern ECLRTL_API void rtlRowset2DictionaryX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
-
-extern ECLRTL_API size32_t rtlDeserializeRow(size32_t lenOut, void * out, IOutputRowDeserializer * deserializer, const void * src);
-extern ECLRTL_API size32_t rtlSerializeRow(size32_t lenOut, void * out, IOutputRowSerializer * serializer, const void * src);
 extern ECLRTL_API size32_t rtlDeserializeToBuilder(ARowBuilder & builder, IOutputRowDeserializer * deserializer, const void * src);
 extern ECLRTL_API size32_t rtlSerializeToBuilder(ARowBuilder & builder, IOutputRowSerializer * serializer, const void * src);
 
+//Dictionary serialization/deserialization
+extern ECLRTL_API void rtlDeserializeDictionary(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src);
+extern ECLRTL_API void rtlDeserializeDictionaryFromDataset(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, IHThorHashLookupInfo & hashInfo, size32_t lenSrc, const void * src);
+extern ECLRTL_API void rtlSerializeDictionary(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+extern ECLRTL_API void rtlSerializeDictionaryToDataset(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+
+extern ECLRTL_API void rtlSerializeDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+extern ECLRTL_API void rtlSerializeDictionaryToDataset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+
+//Dictionary serialization/deserialization to a stream - includes length prefix
+extern ECLRTL_API void rtlDeserializeChildDictionary(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, IRowDeserializerSource & in);
+extern ECLRTL_API void rtlDeserializeChildDictionaryFromDataset(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, IHThorHashLookupInfo & hashInfo, IRowDeserializerSource & in);
+extern ECLRTL_API void rtlSerializeChildDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+extern ECLRTL_API void rtlSerializeChildDictionaryToDataset(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+
 //---------------------------------------------------------------------------
 
 class ECLRTL_API ARtlDatasetCursor

rtl/eclrtl/rtlfield.cpp

@@ -74,16 +74,6 @@ size32_t RtlTypeInfoBase::toXML(const byte * self, const byte * selfrow, const R
     return 0;
 }
 
-void RtlTypeInfoBase::serialize(IRtlFieldTypeSerializer & out) const 
-{
-    rtlFailUnexpected();
-}
-
-void RtlTypeInfoBase::deserialize(IRtlFieldTypeDeserializer & in) 
-{
-    rtlFailUnexpected();
-}
-
 const char * RtlTypeInfoBase::queryLocale() const 
 {
     return NULL; 

rtl/eclrtl/rtlfield_imp.hpp

@@ -28,8 +28,6 @@ struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
-    virtual void serialize(IRtlFieldTypeSerializer & out) const;
-    virtual void deserialize(IRtlFieldTypeDeserializer & in);
 
     virtual const char * queryLocale() const;
     virtual const RtlFieldInfo * const * queryFields() const;

rtl/include/eclhelper.hpp

@@ -30,8 +30,8 @@ It should only contain pure interface definitions or inline functions.
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      140
-#define MIN_ACTIVITY_INTERFACE_VERSION  138             //minimum value that is compatible with current interface - without using selectInterface
+#define ACTIVITY_INTERFACE_VERSION      141
+#define MIN_ACTIVITY_INTERFACE_VERSION  141             //minimum value that is compatible with current interface - without using selectInterface
 
 typedef unsigned char byte;
 
@@ -211,8 +211,10 @@ interface IEngineRowAllocator : extends IInterface
     virtual IOutputMetaData * queryOutputMeta() = 0;
     virtual unsigned queryActivityId() = 0;
     virtual StringBuffer &getId(StringBuffer &) = 0;
-    virtual IOutputRowSerializer *createRowSerializer(ICodeContext *ctx = NULL) = 0;
-    virtual IOutputRowDeserializer *createRowDeserializer(ICodeContext *ctx) = 0;
+    virtual IOutputRowSerializer *createDiskSerializer(ICodeContext *ctx = NULL) = 0;
+    virtual IOutputRowDeserializer *createDiskDeserializer(ICodeContext *ctx) = 0;
+    virtual IOutputRowSerializer *createInternalSerializer(ICodeContext *ctx = NULL) = 0;
+    virtual IOutputRowDeserializer *createInternalDeserializer(ICodeContext *ctx) = 0;
 };
 
 interface IRowSerializerTarget
@@ -295,8 +297,6 @@ interface RtlITypeInfo
     virtual size32_t size(const byte * self, const byte * selfrow) const = 0;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const = 0;  // returns the size
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & out) const = 0;
-    virtual void serialize(IRtlFieldTypeSerializer & out) const = 0;
-    virtual void deserialize(IRtlFieldTypeDeserializer & in) = 0;
 
     virtual const char * queryLocale() const = 0;
     virtual const RtlFieldInfo * const * queryFields() const = 0;               // null terminated list
@@ -360,9 +360,13 @@ enum
     MDFgrouped              = 0x0001,
     MDFhasxml               = 0x0002,
     MDFneeddestruct         = 0x0004,
-    MDFneedserialize        = 0x0008,
+    MDFneedserializedisk    = 0x0008,
     MDFunknownmaxlength     = 0x0010,               // max length couldn't be determined from the record structure
     MDFhasserialize         = 0x0020,
+    MDFneedserializeinternal= 0x0040,
+    MDFdiskmatchesinternal  = 0x0080,
+
+    MDFneedserializemask    = (MDFneedserializedisk|MDFneedserializeinternal),
 };
 
 interface IIndirectMemberVisitor
@@ -382,14 +386,18 @@ interface IOutputMetaData : public IRecordSize
     virtual unsigned getVersion() const = 0;
     virtual unsigned getMetaFlags() = 0;
     virtual const RtlTypeInfo * queryTypeInfo() const { return NULL; }                                          // non null for meta from generated code
-    virtual IOutputMetaData * querySerializedMeta() = 0;
 
     virtual void destruct(byte * self) = 0;
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId) = 0;        // ctx is currently allowed to be NULL
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId) = 0;
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId) = 0;
-    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}            // from and to are *hints* for the range of fields to call through with
 
+    virtual IOutputMetaData * querySerializedDiskMeta() = 0;
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) = 0;        // ctx is currently allowed to be NULL
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) = 0;
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId) = 0;
+
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) = 0;        // ctx is currently allowed to be NULL
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) = 0;
+
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}            // from and to are *hints* for the range of fields to call through with
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) = 0;
 };
 

rtl/include/eclhelper_base.hpp

@@ -150,27 +150,39 @@ public:
     virtual unsigned getMetaFlags()                         { return MDFhasserialize|MDFhasxml; }
 
     virtual void destruct(byte * self)                      {}
-    virtual IOutputMetaData * querySerializedMeta()         { return this; }
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputMetaData * querySerializedDiskMeta()    { return this; }
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)
     {
         return new CVariableOutputRowSerializer(activityId, this);
     }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId)
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId)
     {
-        ISourceRowPrefetcher * fetcher = defaultCreateRowPrefetcher(ctx, activityId);
+        ISourceRowPrefetcher * fetcher = defaultCreateDiskPrefetcher(ctx, activityId);
         if (fetcher)
             return fetcher;
         //Worse case implementation using a deserialize
         return new CSimpleSourceRowPrefetcher(*this, ctx, activityId);
     }
-    virtual CSourceRowPrefetcher * createRawRowPrefetcher(unsigned activityId) { return NULL; }
+    //Default internal serializers are the same as the disk versions
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId)
+    {
+        return createDiskSerializer(ctx, activityId);
+    }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId)
+    {
+        return createDiskDeserializer(ctx, activityId);
+    }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) { }
 
-    inline ISourceRowPrefetcher * defaultCreateRowPrefetcher(ICodeContext * ctx, unsigned activityId)
+protected:
+    //This is the prefetch function that is actually generated by the code generator
+    virtual CSourceRowPrefetcher * doCreateDiskPrefetcher(unsigned activityId) { return NULL; }
+
+    inline ISourceRowPrefetcher * defaultCreateDiskPrefetcher(ICodeContext * ctx, unsigned activityId)
     {
-        if (getMetaFlags() & MDFneedserialize)
-            return querySerializedMeta()->createRowPrefetcher(ctx, activityId);
-        CSourceRowPrefetcher * fetcher = createRawRowPrefetcher(activityId);
+        if (getMetaFlags() & MDFneedserializedisk)
+            return querySerializedDiskMeta()->createDiskPrefetcher(ctx, activityId);
+        CSourceRowPrefetcher * fetcher = doCreateDiskPrefetcher(activityId);
         if (fetcher)
         {
             fetcher->onCreate(ctx);
@@ -189,17 +201,17 @@ public:
     virtual size32_t getMinRecordSize() const               { return fixedSize; }
     virtual size32_t getFixedSize() const                   { return fixedSize; }
 
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)
     {
         return new CFixedOutputRowSerializer(activityId, fixedSize);
     }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId)
     {
         return new CFixedOutputRowDeserializer(activityId, fixedSize);
     }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId)
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId)
     {
-        ISourceRowPrefetcher * fetcher = defaultCreateRowPrefetcher(ctx, activityId);
+        ISourceRowPrefetcher * fetcher = defaultCreateDiskPrefetcher(ctx, activityId);
         if (fetcher)
             return fetcher;
         return new CFixedSourceRowPrefetcher(activityId, fixedSize);
@@ -228,15 +240,15 @@ public:
     virtual size32_t getMinRecordSize() const               { return 0; }
     virtual size32_t getFixedSize() const                   { return 0; }  // is pseudo-variable
     virtual void toXML(const byte * self, IXmlWriter & out) { }
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)
     {
         return new CFixedOutputRowSerializer(activityId, 0);
     }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId)
     {
         return new CFixedOutputRowDeserializer(activityId, 0);
     }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId)
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId)
     {
         return new CFixedSourceRowPrefetcher(activityId, 0);
     }

thorlcr/activities/diskread/thdiskread.cpp

@@ -40,7 +40,7 @@ public:
             Owned<IException> e = MakeActivityWarning(&container, TE_GroupMismatch, "DFS and code generated group info. differs: DFS(%s), CodeGen(%s), using DFS info", isGrouped?"grouped":"ungrouped", codeGenGrouped?"grouped":"ungrouped");
             container.queryJob().fireException(e);
         }
-        IOutputMetaData *recordSize = helper->queryDiskRecordSize()->querySerializedMeta();
+        IOutputMetaData *recordSize = helper->queryDiskRecordSize()->querySerializedDiskMeta();
         if (recordSize->isFixedSize()) // fixed size
         {
             if (0 != fileDesc->queryProperties().getPropInt("@recordSize"))

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -69,7 +69,7 @@ public:
         if (_helper)
             baseHelper.set(_helper);
         helper = (IHThorDiskReadArg *)queryHelper();
-        IOutputMetaData *diskRowMeta = queryDiskRowInterfaces()->queryRowMetaData()->querySerializedMeta();
+        IOutputMetaData *diskRowMeta = queryDiskRowInterfaces()->queryRowMetaData()->querySerializedDiskMeta();
         isFixedDiskWidth = diskRowMeta->isFixedSize();
         diskRowMinSz = diskRowMeta->getMinRecordSize();
         helper->createSegmentMonitors(this);

thorlcr/activities/diskwrite/thdiskwrite.cpp

@@ -36,7 +36,7 @@ public:
         CWriteMasterBase::init();
 
         IHThorDiskWriteArg *helper=(IHThorDiskWriteArg *)queryHelper();
-        IOutputMetaData *irecsize = helper->queryDiskRecordSize()->querySerializedMeta();
+        IOutputMetaData *irecsize = helper->queryDiskRecordSize()->querySerializedDiskMeta();
         IPropertyTree &props = fileDesc->queryProperties();
         if (0 != (helper->getFlags() & TDXgrouped))
             props.setPropBool("@grouped", true);

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -749,7 +749,7 @@ public:
         pullBufferSize = DISTRIBUTE_PULL_BUFFER_SIZE;
         selfstopped = false;
         pull = false;
-        fixedEstSize = meta->querySerializedMeta()->getFixedSize();
+        fixedEstSize = meta->querySerializedDiskMeta()->getFixedSize();
         rowManager = activity->queryJob().queryRowManager();
 
         allowSpill = activity->getOptBool(THOROPT_HDIST_SPILL, true);

thorlcr/activities/indexread/thindexread.cpp

@@ -155,7 +155,7 @@ protected:
                 if (!keyIndex)
                     throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", indexBaseHelper->getFileName());
             
-                unsigned maxSize = indexBaseHelper->queryDiskRecordSize()->querySerializedMeta()->getRecordSize(NULL); // used only if fixed
+                unsigned maxSize = indexBaseHelper->queryDiskRecordSize()->querySerializedDiskMeta()->getRecordSize(NULL); // used only if fixed
                 Owned <IKeyManager> tlk = createKeyManager(keyIndex, maxSize, NULL);
                 indexBaseHelper->createSegmentMonitors(tlk);
                 tlk->finishSegmentMonitors();

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -166,7 +166,7 @@ public:
     {
         helper = (IHThorIndexReadBaseArg *)container->queryHelper();
         localKey = false;
-        fixedDiskRecordSize = helper->queryDiskRecordSize()->querySerializedMeta()->getFixedSize(); // 0 if variable and unused
+        fixedDiskRecordSize = helper->queryDiskRecordSize()->querySerializedDiskMeta()->getFixedSize(); // 0 if variable and unused
         progress = 0;
         reInit = 0 != (helper->getFlags() & (TIRvarfilename|TIRdynamicfilename));
     }

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -165,7 +165,7 @@ public:
                 }
             }
         }
-        assertex(!(helper->queryDiskRecordSize()->getMetaFlags() & MDFneedserialize));
+        assertex(!(helper->queryDiskRecordSize()->getMetaFlags() & MDFneedserializedisk));
         maxDiskRecordSize = helper->queryDiskRecordSize()->isVariableSize() ? KEYBUILD_MAXLENGTH : helper->queryDiskRecordSize()->getFixedSize();
         // NB: the max [ecl] length is not used, other than setting a max field in the header.
         // However, legacy systems (<=702) check that query rec length == key rec len.

thorlcr/graph/thgraph.cpp

@@ -2772,7 +2772,7 @@ IOutputRowSerializer * CActivityBase::queryRowSerializer()
 {
     if (CABserializerlock.lock()) {
         if (!rowSerializer)
-            rowSerializer.setown(queryRowMetaData()->createRowSerializer(queryCodeContext(),queryActivityId()));
+            rowSerializer.setown(queryRowMetaData()->createDiskSerializer(queryCodeContext(),queryActivityId()));
         CABserializerlock.unlock();
     }
     return rowSerializer;
@@ -2782,7 +2782,7 @@ IOutputRowDeserializer * CActivityBase::queryRowDeserializer()
 {
     if (CABdeserializerlock.lock()) {
         if (!rowDeserializer)
-            rowDeserializer.setown(queryRowMetaData()->createRowDeserializer(queryCodeContext(),queryActivityId()));
+            rowDeserializer.setown(queryRowMetaData()->createDiskDeserializer(queryCodeContext(),queryActivityId()));
         CABdeserializerlock.unlock();
     }
     return rowDeserializer;

thorlcr/msort/tsortl.cpp

@@ -187,10 +187,10 @@ public:
         socket->set_block_mode(BF_SYNC_TRANSFER_PULL,0,DEFAULTTIMEOUT*1000);
         stopped = false;
         initbuf = true;
-        size32_t maxsz = rowif->queryRowMetaData()->querySerializedMeta()->getRecordSize(NULL);
+        size32_t maxsz = rowif->queryRowMetaData()->querySerializedDiskMeta()->getRecordSize(NULL);
         // bit of a guess - TBD better (using ISerialStream?)
-        if (maxsz<rowif->queryRowMetaData()->querySerializedMeta()->getMinRecordSize())
-            maxsz=rowif->queryRowMetaData()->querySerializedMeta()->getMinRecordSize();
+        if (maxsz<rowif->queryRowMetaData()->querySerializedDiskMeta()->getMinRecordSize())
+            maxsz=rowif->queryRowMetaData()->querySerializedDiskMeta()->getMinRecordSize();
 
         if (maxsz>bufsize)
             preallocated = maxsz;

thorlcr/thorutil/thbuf.cpp

@@ -1395,7 +1395,7 @@ class CSharedWriteAheadDisk : public CSharedWriteAheadBase
     }
 public:
     CSharedWriteAheadDisk(CActivityBase *activity, const char *spillName, unsigned outputCount, IRowInterfaces *rowIf, IDiskUsage *_iDiskUsage) : CSharedWriteAheadBase(activity, outputCount, rowIf),
-        allocator(rowIf->queryRowAllocator()), serializer(rowIf->queryRowSerializer()), deserializer(rowIf->queryRowDeserializer()), serializeMeta(meta->querySerializedMeta()), iDiskUsage(_iDiskUsage)
+        allocator(rowIf->queryRowAllocator()), serializer(rowIf->queryRowSerializer()), deserializer(rowIf->queryRowDeserializer()), serializeMeta(meta->querySerializedDiskMeta()), iDiskUsage(_iDiskUsage)
     {
         assertex(spillName);
         spillFile.setown(createIFile(spillName));

thorlcr/thorutil/thmem.cpp

@@ -1817,8 +1817,10 @@ class COutputMetaWithChildRow : public CSimpleInterface, implements IOutputMetaD
     Linked<IEngineRowAllocator> childAllocator;
     IOutputMetaData *childMeta;
     size32_t extraSz;
-    Owned<IOutputRowSerializer> serializer;
-    Owned<IOutputRowDeserializer> deserializer;
+    Owned<IOutputRowSerializer> diskSerializer;
+    Owned<IOutputRowDeserializer> diskDeserializer;
+    Owned<IOutputRowSerializer> internalSerializer;
+    Owned<IOutputRowDeserializer> internalDeserializer;
     Owned<ISourceRowPrefetcher> prefetcher;
 
     class CSerializer : public CSimpleInterface, implements IOutputRowSerializer
@@ -1924,25 +1926,37 @@ public:
     {
         OwnedConstThorRow childRow = *(const void **)(self+extraSz);
     }
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)
     {
-        if (!serializer)
-            serializer.setown(new CSerializer(childMeta->createRowSerializer(ctx, activityId), extraSz));
-        return LINK(serializer);
+        if (!diskSerializer)
+            diskSerializer.setown(new CSerializer(childMeta->createDiskSerializer(ctx, activityId), extraSz));
+        return LINK(diskSerializer);
     }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId)
     {
-        if (!deserializer)
-            deserializer.setown(new CDeserializer(childMeta->createRowDeserializer(ctx, activityId), childAllocator, extraSz));
-        return LINK(deserializer);
+        if (!diskDeserializer)
+            diskDeserializer.setown(new CDeserializer(childMeta->createDiskDeserializer(ctx, activityId), childAllocator, extraSz));
+        return LINK(diskDeserializer);
     }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId)
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId)
     {
         if (!prefetcher)
-            prefetcher.setown(new CPrefetcher(childMeta->createRowPrefetcher(ctx, activityId), extraSz));
+            prefetcher.setown(new CPrefetcher(childMeta->createDiskPrefetcher(ctx, activityId), extraSz));
         return LINK(prefetcher);
     }
-    virtual IOutputMetaData * querySerializedMeta() { return this; }
+    virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId)
+    {
+        if (!internalSerializer)
+            internalSerializer.setown(new CSerializer(childMeta->createInternalSerializer(ctx, activityId), extraSz));
+        return LINK(internalSerializer);
+    }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId)
+    {
+        if (!internalDeserializer)
+            internalDeserializer.setown(new CDeserializer(childMeta->createInternalDeserializer(ctx, activityId), childAllocator, extraSz));
+        return LINK(internalDeserializer);
+    }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) 
     {
         //GH: I think this is what it should do, please check
@@ -1960,19 +1974,21 @@ class COutputMetaWithExtra : public CSimpleInterface, implements IOutputMetaData
 {
     Linked<IOutputMetaData> meta;
     size32_t metaSz;
-    Owned<IOutputRowSerializer> serializer;
-    Owned<IOutputRowDeserializer> deserializer;
+    Owned<IOutputRowSerializer> diskSerializer;
+    Owned<IOutputRowDeserializer> diskDeserializer;
+    Owned<IOutputRowSerializer> internalSerializer;
+    Owned<IOutputRowDeserializer> internalDeserializer;
     Owned<ISourceRowPrefetcher> prefetcher;
     Owned<IOutputMetaData> serializedmeta;
 
-    class CSerialization : public CSimpleInterface, implements IOutputRowSerializer
+    class CSerializer : public CSimpleInterface, implements IOutputRowSerializer
     {
         Owned<IOutputRowSerializer> serializer;
         size32_t metaSz;
     public:
         IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-        CSerialization(IOutputRowSerializer *_serializer, size32_t _metaSz) : serializer(_serializer), metaSz(_metaSz)
+        CSerializer(IOutputRowSerializer *_serializer, size32_t _metaSz) : serializer(_serializer), metaSz(_metaSz)
         {
         }
         virtual void serialize(IRowSerializerTarget &out, const byte *self)
@@ -2047,33 +2063,45 @@ public:
 //The following can only be called if getMetaDataVersion >= 1, may seh otherwise.  Creating a different interface was too painful
     virtual unsigned getMetaFlags() { return meta->getMetaFlags(); }
     virtual void destruct(byte * self) { meta->destruct(self); }
-    virtual IOutputRowSerializer * createRowSerializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId)
     {
-        if (!serializer)
-            serializer.setown(new CSerialization(meta->createRowSerializer(ctx, activityId), metaSz));
-        return LINK(serializer);
+        if (!diskSerializer)
+            diskSerializer.setown(new CSerializer(meta->createDiskSerializer(ctx, activityId), metaSz));
+        return LINK(diskSerializer);
     }
-    virtual IOutputRowDeserializer * createRowDeserializer(ICodeContext * ctx, unsigned activityId)
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId)
     {
-        if (!deserializer)
-            deserializer.setown(new CDeserializer(meta->createRowDeserializer(ctx, activityId), metaSz));
-        return LINK(deserializer);
+        if (!diskDeserializer)
+            diskDeserializer.setown(new CDeserializer(meta->createDiskDeserializer(ctx, activityId), metaSz));
+        return LINK(diskDeserializer);
     }
-    virtual ISourceRowPrefetcher * createRowPrefetcher(ICodeContext * ctx, unsigned activityId)
+    virtual ISourceRowPrefetcher * createDiskPrefetcher(ICodeContext * ctx, unsigned activityId)
     {
         if (!prefetcher)
-            prefetcher.setown(new CPrefetcher(meta->createRowPrefetcher(ctx, activityId), metaSz));
+            prefetcher.setown(new CPrefetcher(meta->createDiskPrefetcher(ctx, activityId), metaSz));
         return LINK(prefetcher);
     }
-    virtual IOutputMetaData * querySerializedMeta() 
+    virtual IOutputMetaData * querySerializedDiskMeta() 
     { 
-        IOutputMetaData *sm = meta->querySerializedMeta();
+        IOutputMetaData *sm = meta->querySerializedDiskMeta();
         if (sm==meta.get())
             return this;
         if (!serializedmeta.get())
             serializedmeta.setown(new COutputMetaWithExtra(sm,metaSz));
         return serializedmeta.get();
     } 
+    virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId)
+    {
+        if (!internalSerializer)
+            internalSerializer.setown(new CSerializer(meta->createInternalSerializer(ctx, activityId), metaSz));
+        return LINK(internalSerializer);
+    }
+    virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId)
+    {
+        if (!internalDeserializer)
+            internalDeserializer.setown(new CDeserializer(meta->createInternalDeserializer(ctx, activityId), metaSz));
+        return LINK(internalDeserializer);
+    }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor)
     {
         meta->walkIndirectMembers(self, visitor);