HPCC-18021 New record translation engine

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -410,7 +410,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
       if (GENERATE_COVERAGE_INFO)
         message ("Build system with coverage.")
-        SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
+        if (CMAKE_COMPILER_IS_CLANGXX)
+          SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+        else()
+          SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
+        endif()
       endif()
       # Handle forced default char type
       if (USE_SIGNED_CHAR AND USE_UNSIGNED_CHAR )

ecl/hql/hqlutil.cpp

@@ -8696,7 +8696,10 @@ bool ConstantRowCreator::processFieldValue(IHqlExpression * optLhs, ITypeInfo *
             return false;
         unsigned orig = out.length();
         void *tgt = out.reserve(9);
-        rtlSetPackedUnsigned(tgt, rhs->queryValue()->getIntValue());
+        if (lhsType->isSigned())
+            rtlSetPackedSigned(tgt, rhs->queryValue()->getIntValue());
+        else
+            rtlSetPackedUnsigned(tgt, rhs->queryValue()->getIntValue());
         unsigned actualSize = rtlGetPackedSize(tgt);
         out.setLength(orig+actualSize);
         return true;
@@ -8831,8 +8834,8 @@ bool ConstantRowCreator::processFieldValue(IHqlExpression * optLhs, ITypeInfo *
         }
         else
         {
-            UChar * target = (UChar *) out.reserve(sizeLhs*sizeof(UChar));
-            for (size32_t pos = 0; pos < sizeLhs; pos++)
+            UChar * target = (UChar *) out.reserve(sizeLhs);
+            for (size32_t pos = 0; pos < sizeLhs/sizeof(UChar); pos++)
                 target[pos] = (UChar) ' ';
             castValue->toMem(target);
         }

rtl/eclrtl/rtlds_imp.hpp

@@ -161,7 +161,7 @@ protected:
 
 private:
     //Force errors....
-    inline rtlRowAttr(const rtlRowAttr &) {}
+    inline rtlRowAttr(const rtlRowAttr &) { row = NULL; }
     inline rtlRowAttr & operator = (const rtlRowAttr & other) { row = NULL; return *this; }
 
 protected:
@@ -195,7 +195,7 @@ protected:
 
 private:
     //Force errors....
-    inline rtlRowsAttr(const rtlRowsAttr &) {}
+    inline rtlRowsAttr(const rtlRowsAttr &) { count = 0; rows = NULL; }
     inline rtlRowsAttr & operator = (const rtlRowsAttr & other) { count = 0; rows = NULL; return *this; }
 
 public:

rtl/eclrtl/rtldynfield.cpp

@@ -24,6 +24,9 @@
 #include "eclrtl_imp.hpp"
 #include "rtldynfield.hpp"
 #include "rtlrecord.hpp"
+#include "rtlembed.hpp"
+
+//#define TRACE_TRANSLATION
 
 //---------------------------------------------------------------------------------------------------------------------
 
@@ -497,8 +500,7 @@ extern ECLRTL_API void getFieldVal(size32_t & __lenResult,char * & __result, int
             unsigned numOffsets = r->getNumVarFields() + 1;
             size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
             RtlRow offsetCalculator(*r, row, numOffsets, variableOffsets);
-            if (column != (unsigned) -1)
-                offsetCalculator.getUtf8(__lenResult, __result, column);
+            offsetCalculator.getUtf8(__lenResult, __result, column);
         }
     }
 }
@@ -510,3 +512,601 @@ extern ECLRTL_API int getFieldNum(const char *fieldName, IOutputMetaData &  meta
         return r->getFieldNum(fieldName);
     return -1;
 }
+enum FieldMatchType {
+    // On a field, exactly one of the above is set, but translator returns a bitmap indicating
+    // which were required (and we can restrict translation to allow some types but not others)
+    match_perfect     = 0x00,    // exact type match - use memcpy
+    match_link        = 0x01,    // copy a nested dataset by linking
+    match_move        = 0x02,    // at least one field has moved (set on translator)
+    match_remove      = 0x04,    // at least one field has been removed (set on translator)
+    match_truncate    = 0x08,    // dest is truncated copy of source - use memcpy
+    match_extend      = 0x10,    // dest is padded version of source - use memcpy and memset
+    match_typecast    = 0x20,    // type has changed - cast required
+    match_none        = 0x40,    // No matching field in source - use null value
+    match_recurse     = 0x80,    // Use recursive translator for child records/datasets
+    match_fail        = 0x100,   // no translation possible
+};
+
+StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
+{
+    if (flags == match_perfect)
+        return out.append("perfect");
+    unsigned origlen = out.length();
+    if (flags & match_link) out.append("|link");
+    if (flags & match_move) out.append("|move");
+    if (flags & match_remove) out.append("|remove");
+    if (flags & match_truncate) out.append("|truncate");
+    if (flags & match_extend) out.append("|extend");
+    if (flags & match_typecast) out.append("|typecast");
+    if (flags & match_none) out.append("|none");
+    if (flags & match_recurse) out.append("|recurse");
+    if (flags & match_fail) out.append("|fail");
+    assertex(out.length() > origlen);
+    return out.remove(origlen, 1);
+}
+
+inline constexpr FieldMatchType operator|(FieldMatchType a, FieldMatchType b) { return (FieldMatchType)((int)a | (int)b); }
+inline FieldMatchType &operator|=(FieldMatchType &a, FieldMatchType b) { return (FieldMatchType &) ((int &)a |= (int)b); }
+
+class GeneralRecordTranslator
+{
+public:
+    GeneralRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)
+    : destRecInfo(_destRecInfo), sourceRecInfo(_srcRecInfo)
+    {
+        matchInfo = new MatchInfo[destRecInfo.getNumFields()];
+        createMatchInfo();
+    }
+    ~GeneralRecordTranslator()
+    {
+        delete [] matchInfo;
+    }
+    void describe(unsigned indent = 0) const
+    {
+        for (unsigned idx = 0; idx <  destRecInfo.getNumFields(); idx++)
+        {
+            const char *source = destRecInfo.queryName(idx);
+            const MatchInfo &match = matchInfo[idx];
+            if (match.matchType == match_none)
+                DBGLOG("%*sNo match for field %s - default value will be used", indent, "", source);
+            else
+            {
+                StringBuffer matchStr;
+                DBGLOG("%*sMatch (%s) to field %d for field %s", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source);
+                if (match.subTrans)
+                    match.subTrans->describe(indent+2);
+            }
+        }
+        if (!canTranslate())
+            DBGLOG("%*sTranslation is NOT possible", indent, "");
+        else if (needsTranslate())
+        {
+            StringBuffer matchStr;
+            DBGLOG("%*sTranslation is possible (%s)", indent, "", describeFlags(matchStr, matchFlags).str());
+        }
+        else
+            DBGLOG("%*sTranslation is not necessary", indent, "");
+    }
+    size32_t translate(ARowBuilder &builder, size32_t offset, const byte *sourceRec) const
+    {
+        unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
+        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets);
+        size32_t estimate = destRecInfo.getFixedSize();
+        if (!estimate)
+        {
+            estimate = estimateNewSize(sourceRow);
+            builder.ensureCapacity(offset+estimate, "record");
+        }
+        size32_t origOffset = offset;
+        for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
+        {
+            const RtlFieldInfo *field = destRecInfo.queryField(idx);
+            const RtlTypeInfo *type = field->type;
+            const MatchInfo &match = matchInfo[idx];
+            if (match.matchType == match_none || match.matchType==match_fail)
+                offset = type->buildNull(builder, offset, field);
+            else
+            {
+                unsigned matchField = match.matchIdx;
+                const RtlTypeInfo *sourceType = sourceRecInfo.queryType(matchField);
+                size_t sourceOffset = sourceRow.getOffset(matchField);
+                const byte *source = sourceRec + sourceOffset;
+                size_t copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
+                switch (match.matchType)
+                {
+                case match_perfect:
+                {
+                    // Look ahead for other perfect matches and combine the copies
+                    while (idx < destRecInfo.getNumFields()-1)
+                    {
+                        const MatchInfo &nextMatch = matchInfo[idx+1];
+                        if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
+                        {
+                            idx++;
+                            matchField++;
+                        }
+                        else
+                            break;
+                    }
+                    size_t copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
+                    builder.ensureCapacity(offset+copySize, field->name);
+                    memcpy(builder.getSelf()+offset, source, copySize);
+                    offset += copySize;
+                    break;
+                }
+                case match_truncate:
+                {
+                    assert(type->isFixedSize());
+                    size32_t copySize = type->getMinSize();
+                    builder.ensureCapacity(offset+copySize, field->name);
+                    memcpy(builder.getSelf()+offset, source, copySize);
+                    offset += copySize;
+                    break;
+                }
+                case match_extend:
+                {
+                    assert(type->isFixedSize());
+                    size32_t destSize = type->getMinSize();
+                    builder.ensureCapacity(offset+destSize, field->name);
+                    memcpy(builder.getSelf()+offset, source, copySize);
+                    offset += copySize;
+                    unsigned fillSize = destSize - copySize;
+                    memset(builder.getSelf()+offset, match.fillChar, fillSize);
+                    offset += fillSize;
+                    break;
+                }
+                case match_typecast:
+                    offset = translateScalar(builder, offset, field, type, sourceType, source);
+                    break;
+                case match_link:
+                {
+                    // a 32-bit record count, and a (linked) pointer to an array of record pointers
+                    byte *dest = builder.ensureCapacity(offset+sizeof(size32_t)+sizeof(byte **), field->name)+offset;
+                    *(size32_t *)dest = *(size32_t *)source;
+                    *(byte ***)(dest + sizeof(size32_t)) = rtlLinkRowset(*(byte ***)(source + sizeof(size32_t)));
+                    offset += sizeof(size32_t)+sizeof(byte **);
+                    break;
+                }
+                case match_recurse:
+                    if (type->getType()==type_record)
+                        offset = match.subTrans->translate(builder, offset, source);
+                    else if (type->isLinkCounted())
+                    {
+                        // a 32-bit record count, and a pointer to an array of record pointers
+                        IEngineRowAllocator *childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
+                        assertex(childAllocator);  // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
+
+                        size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
+                        builder.ensureCapacity(offset+sizeInBytes, field->name);
+                        size32_t numRows = 0;
+                        byte **childRows = nullptr;
+                        if (sourceType->isLinkCounted())
+                        {
+                            // a 32-bit count, then a pointer to the source rows
+                            size32_t childCount = *(size32_t *) source;
+                            source += sizeof(size32_t);
+                            const byte ** sourceRows = *(const byte***) source;
+                            for (size32_t childRow = 0; childRow < childCount; childRow++)
+                            {
+                                RtlDynamicRowBuilder childBuilder(*childAllocator);
+                                size32_t childLen = match.subTrans->translate(childBuilder, 0, sourceRows[childRow]);
+                                childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
+                            }
+                        }
+                        else
+                        {
+                            // a 32-bit size, then rows inline
+                            size32_t childSize = *(size32_t *) source;
+                            source += sizeof(size32_t);
+                            const byte *initialSource = source;
+                            while ((size_t)(source - initialSource) < childSize)
+                            {
+                                RtlDynamicRowBuilder childBuilder(*childAllocator);
+                                size32_t childLen = match.subTrans->translate(childBuilder, 0, source);
+                                childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
+                                source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
+                            }
+                        }
+                        // Go back in and patch the count, remembering it may have moved
+                        rtlWriteInt4(builder.getSelf()+offset, numRows);
+                        * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
+                        offset += sizeInBytes;
+                    }
+                    else
+                    {
+                        size32_t countOffset = offset;
+                        byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
+                        offset += sizeof(size32_t);
+                        size32_t initialOffset = offset;
+                        *(size32_t *)dest = 0;  // patched below when true figure known
+                        if (sourceType->isLinkCounted())
+                        {
+                            // a 32-bit count, then a pointer to the source rows
+                            size32_t childCount = *(size32_t *) source;
+                            source += sizeof(size32_t);
+                            const byte ** sourceRows = *(const byte***) source;
+                            for (size32_t childRow = 0; childRow < childCount; childRow++)
+                            {
+                                offset = match.subTrans->translate(builder, offset, sourceRows[childRow]);
+                            }
+                        }
+                        else
+                        {
+                            // a 32-bit size, then rows inline
+                            size32_t childSize = *(size32_t *) source;
+                            source += sizeof(size32_t);
+                            const byte *initialSource = source;
+                            while ((size_t)(source - initialSource) < childSize)
+                            {
+                                offset = match.subTrans->translate(builder, offset, source);
+                                source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
+                            }
+                        }
+                        dest = builder.getSelf() + countOffset;  // Note - may have been moved by reallocs since last calculated
+                        *(size32_t *)dest = offset - initialOffset;
+                    }
+                    break;
+                default:
+                    throwUnexpected();
+                }
+            }
+        }
+        if (estimate && offset-origOffset != estimate)
+        {
+            assert(offset-origOffset > estimate);  // Estimate is always supposed to be conservative
+#ifdef TRACE_TRANSLATION
+            DBGLOG("Wrote %u bytes to record (estimate was %u)\n", offset-origOffset, estimate);
+#endif
+        }
+        return offset;
+    }
+    inline FieldMatchType match() const
+    {
+        return matchFlags;
+    }
+    bool canTranslate() const
+    {
+        return (matchFlags & match_fail) == 0;
+    }
+    bool needsTranslate() const
+    {
+        return (matchFlags & ~match_link) != 0;
+    }
+private:
+    const RtlRecord &destRecInfo;
+    const RtlRecord &sourceRecInfo;
+    unsigned fixedDelta = 0;  // total size of all fixed-size source fields that are not matched
+    UnsignedArray unmatched;  // List of all variable-size source fields that are unmatched
+    FieldMatchType matchFlags = match_perfect;
+
+    struct MatchInfo
+    {
+        unsigned matchIdx = 0;
+        FieldMatchType matchType = match_fail;
+        char fillChar = 0;
+        GeneralRecordTranslator *subTrans = nullptr;
+        ~MatchInfo()
+        {
+            delete subTrans;
+        }
+    } *matchInfo;
+
+    static size32_t translateScalar(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo *destType, const RtlTypeInfo *sourceType, const byte *source)
+    {
+        // This code COULD move into rtlfield.cpp?
+        switch(destType->getType())
+        {
+        case type_boolean:
+        case type_int:
+        case type_swapint:
+        case type_packedint:
+            offset = destType->buildInt(builder, offset, field, sourceType->getInt(source));
+            break;
+        case type_real:
+            offset = destType->buildReal(builder, offset, field, sourceType->getReal(source));
+            break;
+        case type_decimal:  // Go via string - not common enough to special-case
+        case type_data:
+        case type_string:
+        case type_varstring:
+        case type_qstring:
+        {
+            size32_t size;
+            rtlDataAttr text;
+            sourceType->getString(size, text.refstr(), source);
+            offset = destType->buildString(builder, offset, field, size, text.getstr());
+            break;
+        }
+        case type_unicode:
+        case type_varunicode:
+        case type_utf8:
+        {
+            size32_t utf8chars;
+            rtlDataAttr utf8Text;
+            sourceType->getUtf8(utf8chars, utf8Text.refstr(), source);
+            offset = destType->buildUtf8(builder, offset, field, utf8chars, utf8Text.getstr());
+            break;
+        }
+        case type_set:
+        {
+            bool isAll = *(bool *) source;
+            source+= sizeof(bool);
+            byte *dest = builder.ensureCapacity(offset+sizeof(bool)+sizeof(size32_t), field->name)+offset;
+            *(size32_t *) (dest + sizeof(bool)) = 0; // Patch later when size known
+            offset += sizeof(bool) + sizeof(size32_t);
+            if (isAll)
+            {
+                *(bool*) dest = true;
+            }
+            else
+            {
+                *(bool*) dest = false;
+                size32_t sizeOffset = offset - sizeof(size32_t);  // Where we need to patch
+                size32_t childSize = *(size32_t *)source;
+                source += sizeof(size32_t);
+                const byte *initialSource = source;
+                size32_t initialOffset = offset;
+                const RtlTypeInfo *destChildType = destType->queryChildType();
+                const RtlTypeInfo *sourceChildType = sourceType->queryChildType();
+                while ((size_t)(source - initialSource) < childSize)
+                {
+                    offset = translateScalar(builder, offset, field, destChildType, sourceChildType, source);
+                    source += sourceChildType->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
+                }
+                dest = builder.getSelf() + sizeOffset;  // Note - man have been moved by reallocs since last calculated
+                *(size32_t *)dest = offset - initialOffset;
+            }
+            break;
+        }
+        default:
+            throwUnexpected();
+        }
+        return offset;
+    }
+
+    size32_t estimateNewSize(const RtlRow &sourceRow) const
+    {
+        //DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
+        size32_t expectedSize = sourceRow.getRecordSize() - fixedDelta;
+        //DBGLOG("Source record size without omitted fixed size fields is %d", expectedSize);
+        ForEachItemIn(i, unmatched)
+        {
+            unsigned fieldNo = unmatched.item(i);
+            expectedSize -= sourceRow.getSize(fieldNo);
+            //DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
+        }
+        if (matchFlags & ~(match_perfect|match_link|match_none|match_extend|match_truncate))
+        {
+            for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
+            {
+                const MatchInfo &match = matchInfo[idx];
+                const RtlTypeInfo *type = destRecInfo.queryType(idx);
+                unsigned matchField = match.matchIdx;
+                switch (match.matchType)
+                {
+                case match_perfect:
+                case match_link:
+                case match_none:
+                case match_extend:
+                case match_truncate:
+                    // These ones were already included in fixedDelta
+                    break;
+                default:
+                    // This errs on the side of small - i.e. it assumes that all typecasts end up at minimum size
+                    // We could do better in some cases e.g. variable string <-> variable unicode we can assume factor of 2,
+                    // uft8 <-> string we could calculate here - but unlikely to be worth the effort.
+                    // But it's fine for fixed size output fields, including truncate/extend
+                    // We could also precalculate the expected delta if all omitted fields are fixed size - but not sure how likely/worthwhile that is.
+                    expectedSize += type->getMinSize() - sourceRow.getSize(matchField);
+                    //DBGLOG("Adjusting estimated size by (%d - %d) to %d for translated field %d (%s)", (int) sourceRow.getSize(matchField), type->getMinSize(), expectedSize, matchField, sourceRecInfo.queryName(matchField));
+                    break;
+                }
+            }
+        }
+        return expectedSize;
+    }
+    void createMatchInfo()
+    {
+        for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
+        {
+            const RtlFieldInfo *field = destRecInfo.queryField(idx);
+            const RtlTypeInfo *type = field->type;
+            MatchInfo &info = matchInfo[idx];
+            info.matchIdx = sourceRecInfo.getFieldNum(destRecInfo.queryName(idx));
+            if (info.matchIdx == -1)
+            {
+                info.matchType = match_none;
+                size32_t defaultSize = field->initializer ? type->size(field->initializer, nullptr) : type->getMinSize();
+                fixedDelta -= defaultSize;
+                //DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
+            }
+            else
+            {
+                const RtlTypeInfo *sourceType = sourceRecInfo.queryType(info.matchIdx);
+                if (!type->isScalar() || !sourceType->isScalar())
+                {
+                    if (type->getType() != sourceType->getType())
+                        info.matchType = match_fail;  // No translation from one non-scalar type to another
+                    else
+                    {
+                        switch (type->getType())
+                        {
+                        case type_set:
+                            if (type->queryChildType()->fieldType==sourceType->queryChildType()->fieldType &&
+                                type->queryChildType()->length==sourceType->queryChildType()->length)
+                                info.matchType = match_perfect;
+                            else
+                                info.matchType = match_typecast;
+                            break;
+                        case type_row:      // These are not expected I think...
+                            throwUnexpected();
+                        case type_record:
+                        case type_table:
+                        {
+                            const RtlRecord *subDest = destRecInfo.queryNested(idx);
+                            const RtlRecord *subSrc = sourceRecInfo.queryNested(info.matchIdx);
+                            info.subTrans = new GeneralRecordTranslator(*subDest, *subSrc);
+                            if (!info.subTrans->needsTranslate())
+                            {
+                                // Child does not require translation, but check linkcount mode matches too!
+                                if (type->isLinkCounted())
+                                    if (sourceType->isLinkCounted())
+                                        info.matchType = match_link;
+                                    else
+                                        info.matchType = match_recurse;
+                                else
+                                    if (sourceType->isLinkCounted())
+                                        info.matchType = match_recurse;
+                                    else
+                                        info.matchType = match_perfect;
+                                if (info.matchType != match_recurse)
+                                {
+                                    delete info.subTrans;
+                                    info.subTrans = nullptr;
+                                }
+                            }
+                            else if (info.subTrans->canTranslate())
+                            {
+                                info.matchType = match_recurse;
+                                matchFlags |= info.subTrans->matchFlags;
+                            }
+                            else
+                                info.matchType = match_fail;
+                            break;
+                        }
+                        default:
+                            info.matchType = match_fail;
+                            break;
+                        }
+                    }
+                }
+                else if (type->fieldType==sourceType->fieldType)
+                {
+                    if (type->length==sourceType->length)
+                    {
+                        info.matchType = match_perfect;
+                    }
+                    else
+                    {
+                        assert(type->isFixedSize());  // Both variable size would have matched length above
+                        info.matchType = match_typecast;
+                        if (type->length < sourceType->length)
+                        {
+                            if (type->canTruncate())
+                            {
+                                info.matchType = match_truncate;
+                                fixedDelta += sourceType->getMinSize()-type->getMinSize();
+                                //DBGLOG("Increasing fixedDelta size by %d to %d for truncated field %d (%s)", sourceType->getMinSize()-type->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
+                            }
+                        }
+                        else
+                        {
+                            if (type->canExtend(info.fillChar))
+                            {
+                                info.matchType = match_extend;
+                                fixedDelta += sourceType->getMinSize()-type->getMinSize();
+                                //DBGLOG("Decreasing fixedDelta size by %d to %d for truncated field %d (%s)", type->getMinSize()-sourceType->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
+                            }
+                        }
+                    }
+                }
+                else
+                    info.matchType = match_typecast;
+                // MORE - could note the highest interesting fieldnumber in the source and not bother filling in offsets after that
+                // Not sure it would help much though - usually need to know the total record size anyway in real life
+                if (idx != info.matchIdx)
+                    matchFlags |= match_move;
+            }
+            matchFlags |= info.matchType;
+        }
+        if (sourceRecInfo.getNumFields() > destRecInfo.getNumFields())
+            matchFlags |= match_remove;
+        if (matchFlags && !destRecInfo.getFixedSize())
+        {
+            for (unsigned idx = 0; idx < sourceRecInfo.getNumFields(); idx++)
+            {
+                const RtlFieldInfo *field = sourceRecInfo.queryField(idx);
+                const RtlTypeInfo *type = field->type;
+                if (destRecInfo.getFieldNum(field->name) == (unsigned) -1)
+                {
+                    // unmatched field
+                    if (type->isFixedSize())
+                    {
+                        //DBGLOG("Reducing estimated size by %d for (fixed size) omitted field %s", (int) type->getMinSize(), field->name);
+                        fixedDelta += type->getMinSize();
+                    }
+                    else
+                        unmatched.append(idx);
+                }
+            }
+            //DBGLOG("Source record contains %d bytes of omitted fixed size fields", fixedDelta);
+        }
+    }
+};
+
+class TranslatedRowStream : public CInterfaceOf<IRowStream>
+{
+public:
+    TranslatedRowStream(IRowStream *_inputStream, IEngineRowAllocator *_resultAllocator, const RtlRecord &outputRecord, const RtlRecord &inputRecord)
+    : inputStream(_inputStream), resultAllocator(_resultAllocator), translator(outputRecord, inputRecord)
+    {
+        translator.describe();
+    }
+    virtual const void *nextRow()
+    {
+        if (eof)
+            return NULL;
+        const void *inRow = inputStream->nextRow();
+        if (!inRow)
+        {
+            if (eogSeen)
+                eof = true;
+            else
+                eogSeen = true;
+            return nullptr;
+        }
+        else
+            eogSeen = false;
+        RtlDynamicRowBuilder rowBuilder(resultAllocator);
+        size32_t len = translator.translate(rowBuilder, 0, (const byte *) inRow);
+        rtlReleaseRow(inRow);
+        return rowBuilder.finalizeRowClear(len);
+    }
+    virtual void stop() override
+    {
+        resultAllocator.clear();
+    }
+    bool canTranslate() const
+    {
+        return translator.canTranslate();
+    }
+    bool needsTranslate() const
+    {
+        return translator.needsTranslate();
+    }
+protected:
+    Linked<IRowStream> inputStream;
+    Linked<IEngineRowAllocator> resultAllocator;
+    const GeneralRecordTranslator translator;
+    unsigned numOffsets = 0;
+    size_t * variableOffsets = nullptr;
+    bool eof = false;
+    bool eogSeen = false;
+};
+
+extern ECLRTL_API IRowStream * transformRecord(IEngineRowAllocator * resultAllocator,IOutputMetaData &  metaInput,IRowStream * input)
+{
+    if (resultAllocator->queryOutputMeta()==&metaInput)
+        return LINK(input);
+    Owned<TranslatedRowStream> stream = new TranslatedRowStream(input, resultAllocator,
+                                                                *resultAllocator->queryOutputMeta()->queryRecordAccessor(true),
+                                                                *metaInput.queryRecordAccessor(true));
+    if (!stream->needsTranslate())
+        return LINK(input);
+    else if (!stream->canTranslate())
+        rtlFail(0, "Cannot translate record stream");
+    else
+        return stream.getClear();
+}
+

rtl/eclrtl/rtldynfield.hpp

@@ -107,5 +107,6 @@ extern ECLRTL_API void getFieldVal(size32_t & __lenResult, char * & __result, in
  */
 extern ECLRTL_API int getFieldNum(const char *fieldName, IOutputMetaData &  metaVal);
 
+extern ECLRTL_API IRowStream * transformRecord(IEngineRowAllocator * resultAllocator,IOutputMetaData &  metaInput,IRowStream * input);
 
 #endif

rtl/eclrtl/rtlfield.hpp

@@ -41,7 +41,17 @@ struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
-
+    virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const;
+    virtual size32_t buildReal(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, double val) const;
+
+    virtual double getReal(const void * ptr) const;
+    virtual bool isScalar() const;
+    virtual bool isNumeric() const { return false; }
+    virtual bool canTruncate() const { return false; }
+    virtual bool canExtend(char &) const { return false; }
 
     virtual const char * queryLocale() const;
     virtual const RtlFieldInfo * const * queryFields() const;
@@ -55,8 +65,10 @@ struct ECLRTL_API RtlBoolTypeInfo : public RtlTypeInfoBase
     inline RtlBoolTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
 };
@@ -66,10 +78,15 @@ struct ECLRTL_API RtlRealTypeInfo : public RtlTypeInfoBase
     inline RtlRealTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildReal(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, double val) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual double getReal(const void * ptr) const;
+    virtual bool isNumeric() const { return true; }
+
 private:
     inline double value(const void * self) const;
 };
@@ -80,10 +97,17 @@ struct ECLRTL_API RtlIntTypeInfo : public RtlTypeInfoBase
     inline RtlIntTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const;
+
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual double getReal(const void * ptr) const;
+    virtual bool canTruncate() const;
+    virtual bool canExtend(char &fillChar) const;
+    virtual bool isNumeric() const { return true; }
 };
 
 struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase
@@ -91,10 +115,17 @@ struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase
     inline RtlSwapIntTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const;
+
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual double getReal(const void * ptr) const;
+    virtual bool canTruncate() const;
+    virtual bool canExtend(char &fillChar) const;
+    virtual bool isNumeric() const { return true; }
 };
 
 struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
@@ -104,10 +135,15 @@ struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const;
+
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual double getReal(const void * ptr) const;
+    virtual bool isNumeric() const { return true; }
 };
 
 struct ECLRTL_API RtlStringTypeInfo : public RtlTypeInfoBase
@@ -117,10 +153,16 @@ struct ECLRTL_API RtlStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual bool canTruncate() const { return isFixedSize(); }
+    virtual bool canExtend(char &fillChar) const;
+
 };
 
 struct ECLRTL_API RtlDataTypeInfo : public RtlTypeInfoBase
@@ -130,10 +172,15 @@ struct ECLRTL_API RtlDataTypeInfo : public RtlTypeInfoBase
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual bool canTruncate() const { return isFixedSize(); }
+    virtual bool canExtend(char &fillChar) const;
+
 };
 
 struct ECLRTL_API RtlVarStringTypeInfo : public RtlTypeInfoBase
@@ -143,10 +190,13 @@ struct ECLRTL_API RtlVarStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual bool canExtend(char &fillChar) const;
 };
 
 struct ECLRTL_API RtlQStringTypeInfo : public RtlTypeInfoBase
@@ -156,10 +206,14 @@ struct ECLRTL_API RtlQStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual bool canExtend(char &fillChar) const;
+
 };
 
 struct ECLRTL_API RtlDecimalTypeInfo : public RtlTypeInfoBase
@@ -169,10 +223,14 @@ struct ECLRTL_API RtlDecimalTypeInfo : public RtlTypeInfoBase
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual double getReal(const void * ptr) const;
 
     size32_t calcSize() const;
 };
@@ -184,6 +242,7 @@ struct ECLRTL_API RtlCharTypeInfo : public RtlTypeInfoBase
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
 };
@@ -196,8 +255,11 @@ public:
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
 
@@ -215,8 +277,11 @@ public:
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
+
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
 
@@ -234,8 +299,10 @@ public:
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
 
@@ -253,20 +320,25 @@ struct ECLRTL_API RtlRecordTypeInfo : public RtlTypeInfoBase
     virtual size32_t getMinSize() const;
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const;
+    virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
     virtual const RtlFieldInfo * const * queryFields() const { return fields; }
+    virtual bool isScalar() const { return false; }
 };
 
 struct ECLRTL_API RtlCompoundTypeInfo : public RtlTypeInfoBase
 {
     inline RtlCompoundTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo * _child) : RtlTypeInfoBase(_fieldType, _length), child(_child) {}
 
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
     virtual const RtlTypeInfo * queryChildType() const { return child; }
+    virtual bool isScalar() const { return false; }
 
     const RtlTypeInfo * child;
 };
@@ -330,8 +402,10 @@ struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual bool isScalar() const { return false; }
 
     virtual const RtlFieldInfo * const * queryFields() const { return fields; }
 };
@@ -345,6 +419,7 @@ struct ECLRTL_API RtlBitfieldTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
 
@@ -361,8 +436,10 @@ struct ECLRTL_API RtlUnimplementedTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const;
     virtual __int64 getInt(const void * ptr) const;
+    virtual bool isScalar() const { return false; }
 };
 
 /*

rtl/eclrtl/rtlrecord.cpp

@@ -24,6 +24,7 @@
 #include "eclrtl_imp.hpp"
 #include "rtlds_imp.hpp"
 #include "rtlrecord.hpp"
+#include "rtldynfield.hpp"
 
 /*
  * Potential different implementations (for all fixed size has no penalty):
@@ -111,7 +112,7 @@ static unsigned countFields(const RtlFieldInfo * const * fields, bool & contains
 }
 
 
-static const RtlFieldInfo * * expandNestedRows(const RtlFieldInfo * * target, const RtlFieldInfo * const * fields)
+static unsigned expandNestedRows(unsigned idx, const char *prefix, const RtlFieldInfo * const * fields, const RtlFieldInfo * * target, const char * *names)
 {
     for (;*fields;fields++)
     {
@@ -121,32 +122,36 @@ static const RtlFieldInfo * * expandNestedRows(const RtlFieldInfo * * target, co
         {
             const RtlFieldInfo * const * nested = type->queryFields();
             if (nested)
-                target = expandNestedRows(target, nested);
+            {
+                StringBuffer newPrefix(prefix);
+                newPrefix.append(cur->name).append('.');
+                idx = expandNestedRows(idx, newPrefix.str(), nested, target, names);
+            }
         }
         else
-            *target++ = cur;
+        {
+            if (prefix)
+            {
+                StringBuffer name(prefix);
+                name.append(cur->name);
+                names[idx] = name.detach();
+            }
+            else
+                names[idx] = nullptr;
+            target[idx++] = cur;
+        }
     }
-    return target;
+    return idx;
 }
 
 class FieldNameToFieldNumMap
 {
 public:
-    FieldNameToFieldNumMap(const RtlFieldInfo * const * fields, bool expand)
+    FieldNameToFieldNumMap(const RtlRecord &record)
     {
-        unsigned idx = 0;
-        if (expand)
-            expandFields(nullptr, idx, fields);
-        else
-        {
-            for (;*fields;fields++)
-            {
-                const RtlFieldInfo * cur = *fields;
-                const RtlTypeInfo * type = cur->type;
-                map.setValue(cur->name, idx);
-                idx++;
-            }
-        }
+        unsigned numFields = record.getNumFields();
+        for (unsigned idx = 0; idx < numFields;idx++)
+            map.setValue(record.queryName(idx), idx);
     }
     unsigned lookup(const char *name) const
     {
@@ -156,38 +161,7 @@ public:
         else
             return (unsigned) -1;
     }
-protected:
-    void expandFields(const char *prefix, unsigned &idx, const RtlFieldInfo * const * fields)
-    {
-        for (;*fields;fields++)
-        {
-            const RtlFieldInfo * cur = *fields;
-            const RtlTypeInfo * type = cur->type;
-            if (type->getType() == type_record)
-            {
-                const RtlFieldInfo * const * nested = type->queryFields();
-                if (nested)
-                {
-                    StringBuffer newPrefix(prefix);
-                    newPrefix.append(cur->name).append('.');
-                    expandFields(newPrefix, idx, nested);
-                }
-            }
-            else if (prefix)
-            {
-                StringBuffer name(prefix);
-                name.append(cur->name);
-                map.setValue(name, idx);
-                idx++;
-            }
-            else
-            {
-                map.setValue(cur->name, idx);
-                idx++;
-            }
-        }
-    }
-    MapStringTo<unsigned> map;
+    MapConstStringTo<unsigned> map;  // Note - does not copy strings - they should all have sufficient lifetime
 };
 
 RtlRecord::RtlRecord(const RtlRecordTypeInfo & record, bool expandFields)
@@ -195,10 +169,11 @@ RtlRecord::RtlRecord(const RtlRecordTypeInfo & record, bool expandFields)
 {
 }
 
-RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : fields(_fields), originalFields(_fields), nameMap(nullptr)
+RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : fields(_fields), originalFields(_fields), names(nullptr), nameMap(nullptr)
 {
     //MORE: Does not cope with ifblocks.
     numVarFields = 0;
+    numTables = 0;
     //Optionally expand out nested rows.
     if (expandFields)
     {
@@ -207,26 +182,41 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
         if (containsNested)
         {
             const RtlFieldInfo * * allocated  = new const RtlFieldInfo * [numFields+1];
+            names = new const char *[numFields];
             fields = allocated;
-            const RtlFieldInfo * * target = expandNestedRows(allocated, originalFields);
-            assertex(target == fields+numFields);
-            *target = nullptr;
+            unsigned idx = expandNestedRows(0, nullptr, originalFields, allocated, names);
+            assertex(idx == numFields);
+            allocated[idx] = nullptr;
         }
     }
     else
+    {
         numFields = countFields(fields);
-
+    }
     for (unsigned i=0; i < numFields; i++)
     {
-        if (!queryType(i)->isFixedSize())
+        const RtlTypeInfo *curType = queryType(i);
+        if (!curType->isFixedSize())
             numVarFields++;
+        if (curType->getType()==type_table || curType->getType()==type_record)
+            numTables++;
     }
 
     fixedOffsets = new size_t[numFields + 1];
     whichVariableOffset = new unsigned[numFields + 1];
     variableFieldIds = new unsigned[numVarFields];
-
+    if (numTables)
+    {
+        nestedTables = new const RtlRecord *[numTables];
+        tableIds = new unsigned[numTables];
+    }
+    else
+    {
+        nestedTables = nullptr;
+        tableIds = nullptr;
+    }
     unsigned curVariable = 0;
+    unsigned curTable = 0;
     size_t fixedOffset = 0;
     for (unsigned i=0;; i++)
     {
@@ -247,17 +237,46 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
             curVariable++;
             fixedOffset = 0;
         }
+        switch (curType->getType())
+        {
+        case type_table:
+            tableIds[curTable] = i;
+            nestedTables[curTable++] = new RtlRecord(curType->queryChildType()->queryFields(), expandFields);
+            break;
+        case type_record:
+            tableIds[curTable] = i;
+            nestedTables[curTable++] = new RtlRecord(curType->queryFields(), expandFields);
+            break;
+        }
     }
 }
 
 
 RtlRecord::~RtlRecord()
 {
+    if (names)
+    {
+        for (unsigned i = 0; i < numFields; i++)
+        {
+            free((char *) names[i]);
+        }
+        delete [] names;
+    }
     if (fields != originalFields)
+    {
         delete [] fields;
+    }
     delete [] fixedOffsets;
     delete [] whichVariableOffset;
     delete [] variableFieldIds;
+    delete [] tableIds;
+
+    if (nestedTables)
+    {
+        for (unsigned i = 0; i < numTables; i++)
+            delete nestedTables[i];
+        delete [] nestedTables;
+    }
     delete nameMap;
 }
 
@@ -286,9 +305,9 @@ size32_t RtlRecord::getMinRecordSize() const
     return minSize;
 }
 
-static const FieldNameToFieldNumMap *setupNameMap(const RtlFieldInfo * const * fields, bool expand, std::atomic<const FieldNameToFieldNumMap *> &aNameMap)
+static const FieldNameToFieldNumMap *setupNameMap(const RtlRecord &record, std::atomic<const FieldNameToFieldNumMap *> &aNameMap)
 {
-    const FieldNameToFieldNumMap *lnameMap = new FieldNameToFieldNumMap(fields, expand);
+    const FieldNameToFieldNumMap *lnameMap = new FieldNameToFieldNumMap(record);
     const FieldNameToFieldNumMap *expected = nullptr;
     if (aNameMap.compare_exchange_strong(expected, lnameMap))
         return lnameMap;
@@ -307,10 +326,27 @@ unsigned RtlRecord::getFieldNum(const char *fieldName) const
     std::atomic<const FieldNameToFieldNumMap *> &aNameMap = reinterpret_cast<std::atomic<const FieldNameToFieldNumMap *> &>(nameMap);
     const FieldNameToFieldNumMap *useMap = aNameMap.load(std::memory_order_relaxed);
     if (!useMap)
-        useMap = setupNameMap(originalFields, originalFields!=fields, aNameMap);
+        useMap = setupNameMap(*this, aNameMap);
     return useMap->lookup(fieldName);
 }
 
+const char *RtlRecord::queryName(unsigned field) const
+{
+    if (names && names[field])
+        return names[field];
+    return fields[field]->name;
+}
+
+const RtlRecord *RtlRecord::queryNested(unsigned fieldId) const
+{
+    // Map goes in wrong direction (for size reasons). We could replace with a hashtable or binsearch but
+    // should not be enough nested tables for it to be worth it;
+    for (unsigned i = 0; i < numTables; i++)
+        if (tableIds[i]==fieldId)
+            return nestedTables[i];
+    return nullptr;
+}
+
 //---------------------------------------------------------------------------------------------------------------------
 
 RtlRow::RtlRow(const RtlRecord & _info, const void * optRow, unsigned numOffsets, size_t * _variableOffsets) : info(_info), variableOffsets(_variableOffsets)
@@ -328,6 +364,20 @@ __int64 RtlRow::getInt(unsigned field) const
     return type->getInt(self + getOffset(field));
 }
 
+double RtlRow::getReal(unsigned field) const
+{
+    const byte * self = reinterpret_cast<const byte *>(row);
+    const RtlTypeInfo * type = info.queryType(field);
+    return type->getReal(self + getOffset(field));
+}
+
+void RtlRow::getString(size32_t & resultLen, char * & result, unsigned field) const
+{
+    const byte * self = reinterpret_cast<const byte *>(row);
+    const RtlTypeInfo * type = info.queryType(field);
+    return type->getString(resultLen, result, self + getOffset(field));
+}
+
 void RtlRow::getUtf8(size32_t & resultLen, char * & result, unsigned field) const
 {
     const byte * self = reinterpret_cast<const byte *>(row);

rtl/eclrtl/rtlrecord.hpp

@@ -192,7 +192,7 @@ protected:
 
 class FieldNameToFieldNumMap;
 
-struct ECLRTL_API RtlRecord
+class ECLRTL_API RtlRecord
 {
 public:
     friend class RtlRow;
@@ -222,17 +222,23 @@ public:
 
     inline unsigned getNumFields() const { return numFields; }
     inline unsigned getNumVarFields() const { return numVarFields; }
+    inline const RtlFieldInfo * queryField(unsigned field) const { return fields[field]; }
     inline const RtlTypeInfo * queryType(unsigned field) const { return fields[field]->type; }
-    inline const char * queryName(unsigned field) const { return fields[field]->name; }
+    const char * queryName(unsigned field) const;
     unsigned getFieldNum(const char *fieldName) const;
+    const RtlRecord *queryNested(unsigned field) const;
 protected:
-    size_t * fixedOffsets;        // fixed portion of the field offsets + 1 extra
+    size_t * fixedOffsets;         // fixed portion of the field offsets + 1 extra
     unsigned * whichVariableOffset;// which variable offset should be added to the fixed
-    unsigned * variableFieldIds;  // map variable field to real field id.
+    unsigned * variableFieldIds;   // map variable field to real field id.
+    unsigned * tableIds;           // map nested table id to real field id.
     unsigned numFields;
     unsigned numVarFields;
+    unsigned numTables;
     const RtlFieldInfo * const * fields;
     const RtlFieldInfo * const * originalFields;
+    const RtlRecord **nestedTables;
+    const char **names;
     mutable const FieldNameToFieldNumMap *nameMap;
 };
 
@@ -242,6 +248,8 @@ public:
     RtlRow(const RtlRecord & _info, const void * optRow, unsigned numOffsets, size_t * _variableOffsets);
 
     __int64 getInt(unsigned field) const;
+    double getReal(unsigned field) const;
+    void getString(size32_t & resultLen, char * & result, unsigned field) const;
     void getUtf8(size32_t & resultLen, char * & result, unsigned field) const;
 
     size_t getOffset(unsigned field) const
@@ -249,6 +257,11 @@ public:
         return info.getOffset(variableOffsets, field);
     }
 
+    size_t getSize(unsigned field) const
+    {
+        return info.getOffset(variableOffsets, field+1) - info.getOffset(variableOffsets, field);
+    }
+
     size_t getRecordSize() const
     {
         return info.getRecordSize(variableOffsets);

rtl/include/eclhelper.hpp

@@ -361,9 +361,16 @@ interface RtlITypeInfo
     virtual const RtlTypeInfo * queryChildType() const = 0;
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const = 0;
+    virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const = 0;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *result) const = 0;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *result) const = 0;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const = 0;
+    virtual size32_t buildReal(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, double val) const = 0;
 
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const = 0;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const = 0;
     virtual __int64 getInt(const void * ptr) const = 0;
+    virtual double getReal(const void * ptr) const = 0;
     virtual size32_t getMinSize() const = 0;
 };
 
@@ -385,7 +392,10 @@ struct RtlTypeInfo : public RtlITypeInfo
     inline unsigned getBitfieldNumBits() const { return (length >> 8) & 0xff; }
     inline unsigned getBitfieldShift() const { return (length >> 16) & 0xff; }
     inline unsigned getType() const { return (fieldType & RFTMkind); }
-
+    virtual bool isScalar() const = 0;
+    virtual bool isNumeric() const = 0;
+    virtual bool canTruncate() const = 0;
+    virtual bool canExtend(char &) const = 0;
 public:
     unsigned fieldType;
     unsigned length;                // for bitfield (int-size, # bits, bitoffset) << 16

system/jlib/jhash.ipp

@@ -68,7 +68,6 @@ class jlib_decl Mapping : extends MappingBase
     MappingKey          key;
 };
 
-
 class jlib_decl AtomBase : public CInterfaceOf<IAtom>
 {
 public:
@@ -173,6 +172,27 @@ class MappingStringTo : extends Atom
      VALUE_T    val;
 };
 
+template <class VALUE_T,class VALINIT>
+class MappingConstStringTo : public CInterfaceOf<IAtom>
+{
+public:
+    MappingConstStringTo(const char *k, VALINIT a) : key(k), val(a), hash(0)
+    { };
+
+    inline VALUE_T &            getValue()      { return val; };
+
+//interface:IMapping
+    virtual const char *        queryStr() const { return key; }
+    virtual const void *        getKey() const { return key; }
+    virtual unsigned            getHash() const { return hash; }
+    virtual void                setHash(unsigned hval) { hash = hval; }
+
+  protected:
+    unsigned hash;
+    const char * key;
+    VALUE_T val;
+};
+
 
 template <class T, unsigned int K> class KeptHashTableOf
  : public KeptHashTable
@@ -326,6 +346,33 @@ class MapStringTo : extends StringMapOf<MAPPING>
     }
 };
 
+template <class VALUE_T, class VALINIT = VALUE_T, class MAPPING = MappingConstStringTo<VALUE_T, VALINIT> >
+class MapConstStringTo : extends StringMapOf<MAPPING>
+{
+    typedef MapConstStringTo<VALUE_T, VALINIT, MAPPING> SELF;
+  public:
+    MapConstStringTo():StringMapOf<MAPPING>(false){};
+    MapConstStringTo(unsigned initsize, bool _ignorecase):StringMapOf<MAPPING>(initsize, _ignorecase){};
+    MapConstStringTo(bool _ignorecase):StringMapOf<MAPPING>(_ignorecase){};
+    VALUE_T *   getValue(const char *k) const
+    {
+       MAPPING * map = SELF::find(k);
+       if (map)
+          return &map->getValue();
+       return NULL;
+    }
+    static inline VALUE_T * mapToValue(IMapping * _map)
+    {
+       MAPPING * map = (MAPPING *)_map;
+       return &map->getValue();
+    }
+    bool        setValue(const char *k, VALINIT v)
+    {
+       MAPPING * map = new MAPPING(k, v);
+       return this->replaceOwn(*map);
+    }
+};
+
 typedef const char * char_ptr;
 typedef MapStringTo<StringAttr, char_ptr> StringAttrMapping;
 

testing/regress/ecl/key/translateFields.xml

@@ -0,0 +1,140 @@
+<Dataset name='Result 1'>
+ <Row><desc>Untranslated        </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><desc>None needed         </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><desc>Removed             </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><desc>Swapped             </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><desc>Switch linkcounts   </desc><subl><Row><sub1>e1  </sub1><sub2>e1a </sub2></Row><Row><sub1>e2  </sub1><sub2>sub2</sub2></Row></subl><sube><Row><sub1>e3  </sub1><sub2>sub2</sub2></Row><Row><sub1>e4  </sub1><sub2>sub2</sub2></Row></sube><subrec><sub1>sub1</sub1><sub2>sub2</sub2></subrec></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><desc>Link child          </desc><subl><Row><sub1>l1  </sub1><sub2>l1a </sub2></Row><Row><sub1>l2  </sub1><sub2>sub2</sub2></Row></subl><sube></sube><subrec><sub1>sub1</sub1><sub2>sub2</sub2></subrec></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><desc>Translate nested    </desc><subl><Row><sub1>t1  </sub1><sub2>sub2</sub2></Row><Row><sub1>t2  </sub1><sub2>sub2</sub2></Row></subl><sube><Row><sub1>t3  </sub1><sub2>sub2</sub2></Row><Row><sub1>t4  </sub1><sub2>sub2</sub2></Row></sube><subrec><sub1>t5  </sub1><sub2>sub2</sub2></subrec></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><desc>Translate set       </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>aa</Item><Item>aa</Item></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+ <Row><desc>Translate all       </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><All/></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+ <Row><desc>Translate empty set </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><desc>Not matched         </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><desc>Multiple adjacent   </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>string01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><desc>Truncate strings    </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>STRING01</s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><desc>Extend strings      </desc><b>true</b><n0>0</n0><n1>1</n1><n2>2</n2><r1>3.14</r1><d1>-3.14</d1><d2>3.14</d2><set1><Item>s1</Item><Item>s2</Item></set1><s1>STRING1 </s1><v1>v1</v1><v2>v2</v2><u1>€unicode</u1><vu1>€vu1</vu1><vu2>€vu2</vu2><da1>3131323233333434</da1><da2>3535363637373838</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><desc>Typecast integer4   </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><desc>Typecast integer4   </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><desc>Typecast integer2   </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><desc>Typecast integer2   </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><desc>Typecast real4      </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8.</Item><Item>9.</Item></set1><s1>10.0    </s1><v1>11.0</v1><v2>12.0</v2><u1>13.0    </u1><vu1>14.0</vu1><vu2>15.0</vu2><da1>31362E30</da1><da2>31372E3000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><desc>Typecast real4      </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8.</Item><Item>9.</Item></set1><s1>10.0    </s1><v1>11.0</v1><v2>12.0</v2><u1>13.0    </u1><vu1>14.0</vu1><vu2>15.0</vu2><da1>31362E30</da1><da2>31372E3020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><desc>Typecast real8      </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8.</Item><Item>9.</Item></set1><s1>10.0    </s1><v1>11.0</v1><v2>12.0</v2><u1>13.0    </u1><vu1>14.0</vu1><vu2>15.0</vu2><da1>31362E30</da1><da2>31372E3000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><desc>Typecast real8      </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8.</Item><Item>9.</Item></set1><s1>10.0    </s1><v1>11.0</v1><v2>12.0</v2><u1>13.0    </u1><vu1>14.0</vu1><vu2>15.0</vu2><da1>31362E30</da1><da2>31372E3020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><desc>Typecast packed  int</desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><desc>Typecast packed  uns</desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><desc>Typecast big_endian </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><desc>Typecast big_endian </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><desc>Typecast decimal4_2 </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><desc>Typecast decimal4_2 </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><desc>Typecast unsigned  d</desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><desc>Typecast unsigned  d</desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><desc>Typecast string     </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><desc>Typecast string     </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 31'>
+ <Row><desc>Typecast varstring  </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 32'>
+ <Row><desc>Typecast varstring  </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 33'>
+ <Row><desc>Typecast string8    </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11      </v1><v2>12   </v2><u1>13      </u1><vu1>14      </vu1><vu2>15   </vu2><da1>3136202020202020</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 34'>
+ <Row><desc>Typecast string8    </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11      </v1><v2>12   </v2><u1>13      </u1><vu1>14      </vu1><vu2>15   </vu2><da1>3136202020202020</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 35'>
+ <Row><desc>Typecast varstring8 </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 36'>
+ <Row><desc>Typecast varstring8 </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 37'>
+ <Row><desc>Typecast unicode    </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 38'>
+ <Row><desc>Typecast unicode    </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 39'>
+ <Row><desc>Typecast varunicode </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 40'>
+ <Row><desc>Typecast varunicode </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 41'>
+ <Row><desc>Typecast unicode8   </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11      </v1><v2>12   </v2><u1>13      </u1><vu1>14      </vu1><vu2>15   </vu2><da1>3136202020202020</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 42'>
+ <Row><desc>Typecast unicode8   </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11      </v1><v2>12   </v2><u1>13      </u1><vu1>14      </vu1><vu2>15   </vu2><da1>3136202020202020</da1><da2>3137202020202020</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 43'>
+ <Row><desc>Typecast varunicode8</desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 44'>
+ <Row><desc>Typecast varunicode8</desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136000000000000</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 45'>
+ <Row><desc>Typecast utf8       </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>
+<Dataset name='Result 46'>
+ <Row><desc>Typecast utf8       </desc><b>true</b><n0>2</n0><n1>3</n1><n2>4</n2><r1>5.0</r1><d1>6</d1><d2>7</d2><set1><Item>8 </Item><Item>9 </Item></set1><s1>10      </s1><v1>11</v1><v2>12</v2><u1>13      </u1><vu1>14</vu1><vu2>15</vu2><da1>3136</da1><da2>3137000000000000</da2><tail>end</tail></Row>
+</Dataset>

testing/regress/ecl/translateFields.ecl

@@ -0,0 +1,397 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+// Test various record translations
+
+// All try to translate to one of these two structures
+
+// for simple cases
+
+dest := RECORD
+  string20 desc       { default('Not matched')};
+  boolean b           { default(true)};
+  unsigned4 n0        { default(0)};
+  unsigned4 n1        { default(1)};
+  unsigned8 n2        { default(2)};
+  real r1             { default(3.14)};
+  decimal4_2 d1       { default(-3.14)};
+  udecimal4_2 d2      { default(3.14)};
+  set of string2 set1 { default(['s1','s2']) };
+  string8 s1          { default('string01') };
+  varstring v1        { default('v1') };
+  varstring5 v2       { default('v2') };
+  unicode8 u1         { default(u'€unicode') };
+  varunicode vu1      { default(u'€vu1') };
+  varunicode5 vu2     { default(u'€vu2') };
+  data da1            { default(d'11223344') };
+  data8 da2           { default(d'55667788') };
+  string tail         { default('end') };
+END;
+
+// for recursive tests
+
+child := RECORD
+  string4 sub1 { default('sub1')};
+  string4 sub2 { default('sub2')};
+END;
+
+parent := RECORD
+  string20 desc;
+  linkcounted dataset(child) subl;
+  embedded dataset(child) sube;
+  child subrec;
+END;
+
+s := SERVICE
+   streamed dataset(dest) stransform(streamed dataset input) : eclrtl,pure,library='eclrtl',entrypoint='transformRecord',passParameterMeta(true);
+   streamed dataset(parent) rtransform(streamed dataset input) : eclrtl,pure,library='eclrtl',entrypoint='transformRecord',passParameterMeta(true);
+END;
+
+// Untranslated (type descriptors match)
+
+untranslated := dataset([{'Untranslated'}], dest);
+
+// None needed (only defaults differ)
+
+dest_nn := RECORD
+  string20 desc       { default('default') };
+  boolean b           { default(true)};
+  unsigned4 n0        { default(0)};
+  unsigned4 n1        { default(1)};
+  unsigned8 n2        { default(2)};
+  real r1             { default(3.14)};
+  decimal4_2 d1       { default(-3.14)};
+  udecimal4_2 d2      { default(3.14)};
+  set of string2 set1 { default(['s1','s2']) };
+  string8 s1          { default('string01') };
+  varstring v1        { default('v1') };
+  varstring5 v2       { default('v2') };
+  unicode8 u1         { default(u'€unicode') };
+  varunicode vu1      { default(u'€vu1') };
+  varunicode5 vu2     { default(u'€vu2') };
+  data da1            { default(d'11223344') };
+  data8 da2           { default(d'55667788') };
+  string tail         { default('end') };
+END;
+
+none_needed := dataset([{'None needed'}], dest_nn);
+
+// Fields removed at end
+
+dest_removed := RECORD
+  string20 desc       { default('Not matched')};
+  boolean b           { default(true)};
+  unsigned4 n0        { default(0)};
+  unsigned4 n1        { default(1)};
+  unsigned8 n2        { default(2)};
+  real r1             { default(3.14)};
+  decimal4_2 d1       { default(-3.14)};
+  udecimal4_2 d2      { default(3.14)};
+  set of string2 set1 { default(['s1','s2']) };
+  string8 s1          { default('string01') };
+  varstring v1        { default('v1') };
+  varstring5 v2       { default('v2') };
+  unicode8 u1         { default(u'€unicode') };
+  varunicode vu1      { default(u'€vu1') };
+  varunicode5 vu2     { default(u'€vu2') };
+  data da1            { default(d'11223344') };
+  data8 da2           { default(d'55667788') };
+  string tail         { default('end') };
+  string gone         { default('gone') };
+END;
+
+removed := dataset([{'Removed'}], dest_removed);
+
+// Fields swapped (only defaults differ)
+
+dest_swapped := RECORD
+  string20 desc       { default('default') };
+  boolean b           { default(true)};
+  unsigned4 n1        { default(1)};
+  unsigned4 n0        { default(0)};
+  unsigned8 n2        { default(2)};
+  real r1             { default(3.14)};
+  decimal4_2 d1       { default(-3.14)};
+  udecimal4_2 d2      { default(3.14)};
+  set of string2 set1 { default(['s1','s2']) };
+  string8 s1          { default('string01') };
+  varstring v1        { default('v1') };
+  varstring5 v2       { default('v2') };
+  unicode8 u1         { default(u'€unicode') };
+  varunicode vu1      { default(u'€vu1') };
+  varunicode5 vu2     { default(u'€vu2') };
+  data da1            { default(d'11223344') };
+  data8 da2           { default(d'55667788') };
+  string tail         { default('end') };
+END;
+
+swapped_only := dataset([{'Swapped'}], dest_swapped);
+
+// Translating embedded to link counted and vice versa
+
+parent1 := RECORD
+  string20 desc;
+  embedded dataset(child) subl;
+  linkcounted dataset(child) sube;
+END;
+
+switch_linkcounts := dataset([{'Switch linkcounts',[{'e1','e1a'},{'e2'}],[{'e3'},{'e4'}]}], parent1);
+
+// Copying linkcounted datasets via link
+
+parent2 := RECORD
+  string20 desc;
+  linkcounted dataset(child) subl;
+END;
+
+translate_link := dataset([{'Link child',[{'l1','l1a'},{'l2'}]}], parent2);
+
+// Translating fields in nested datasets/records
+
+child1 := record
+  string5 sub1 { default('ts1')};
+END;
+
+parent3 := record
+  STRING20 desc;
+  LINKCOUNTED dataset(child1) subl;
+  EMBEDDED dataset(child1) sube;
+  child1 subrec;
+END;
+
+translate_nested := dataset([{'Translate nested', [{'t1  a'},{'t2  a'}],[{'t3  a'},{'t4  a'}],{'t5  a'}}], parent3);
+
+// Translating a set from one type to another
+
+dest_set := record
+  STRING20 desc;
+  set of string3 set1;
+END;
+
+translate_set := dataset([{'Translate set', ['aab','aac']},{'Translate all', ALL},{'Translate empty set', []}], dest_set);
+
+// Translate where no records match
+
+dest_nomatch := record
+  STRING20 notdesc;
+END;
+
+nomatch := dataset([{'No match'}], dest_nomatch);
+
+// Test multiple adjacent matching fields
+
+dest_multi_adjacent := RECORD
+  string20 desc;
+  unsigned4 n1        { default(1)};
+  unsigned8 n2        { default(2)};
+
+  varstring v1        { default('v1') };
+  varstring5 v2       { default('v2') };
+END;
+
+multi_adjacent := dataset([{'Multiple adjacent'}], dest_multi_adjacent);
+
+// Test truncating strings
+
+dest_truncate := RECORD
+  string20 desc;
+  string18 s1          { default('STRING012345') };
+END;
+
+truncate_strings := dataset([{'Truncate strings'}], dest_truncate);
+
+// Test extending strings
+
+dest_extend := RECORD
+  string20 desc;
+  string7 s1           { default('STRING1') };
+END;
+
+extend_strings := dataset([{'Extend strings'}], dest_extend);
+
+// Test typecasts
+
+declare_typecast_from_int(t, name, pname) := MACRO
+#uniquename(dest_typecast_from_int)
+#uniquename(transform_typecast_from_int)
+%dest_typecast_from_int% := RECORD
+  string20 desc;
+
+  t b     { default(1)};
+  t n1    { default(3)};
+  t n0    { default(2)};  // Note swapped
+  t n2    { default(4)};
+  t r1    { default(5)};
+  t d1    { default(6)};
+  t d2    { default(7)};
+  set of t set1 { default([8,9]) };
+  t s1    { default(10) };
+  t v1    { default(11) };
+  t v2    { default(12) };
+  t u1    { default(13) };
+  t vu1   { default(14) };
+  t vu2   { default(15) };
+  t da1   { default(16) };
+  t da2   { default(17) };
+END;
+
+dest %transform_typecast_from_int%(%dest_typecast_from_int% L) := transform
+  self.desc := L.desc;
+  self.b := (boolean) L.b;
+  self.n0 := (unsigned4) L.n0;
+  self.n1 := (unsigned4) L.n1;
+  self.n2 := (unsigned8) L.n2;
+  self.r1 := (real) L.r1;
+  self.d1 := (decimal4_2) L.d1;
+  self.d2 := (udecimal4_2) L.d2;
+  self.set1 := (set of string2) L.set1;
+  self.s1 := (string8) L.s1;
+  self.v1 := (varstring) L.v1;
+  self.v2 := (varstring5) L.v2;
+  self.u1 := (unicode8) L.u1;
+  self.vu1 := (varunicode) L.vu1;
+  self.vu2 := (varunicode5) L.vu2;
+  self.da1 := (data) L.da1;
+  self.da2 := (data8) L.da2;
+  self := [];
+END;
+
+name := dataset([{'Typecast ' + #TEXT(t)}], %dest_typecast_from_int%);
+pname := PROJECT(nofold(name), %transform_typecast_from_int%(LEFT));
+ENDMACRO;
+
+declare_typecast_from_string(t, name, pname) := MACRO
+#uniquename(dest_typecast_from_string)
+#uniquename(transform_typecast_from_string)
+%dest_typecast_from_string% := RECORD
+  string20 desc;
+
+  t b    { default('1')};
+  t n1    { default('3')};
+  t n0    { default('2')};  // Note swapped
+  t n2    { default('4')};
+  t r1    { default('5')};
+  t d1    { default('6')};
+  t d2    { default('7')};
+  set of t set1 { default(['8','9']) };
+  t s1    { default('10') };
+  t v1    { default('11') };
+  t v2    { default('12') };
+  t u1    { default('13') };
+  t vu1   { default('14') };
+  t vu2   { default('15') };
+  t da1   { default('16') };
+  t da2   { default('17') };
+END;
+
+dest %transform_typecast_from_string%(%dest_typecast_from_string% L) := transform
+  self.desc := L.desc;
+  self.b := (boolean) L.b;
+  self.n0 := (unsigned4) L.n0;
+  self.n1 := (unsigned4) L.n1;
+  self.n2 := (unsigned8) L.n2;
+  self.r1 := (real) L.r1;
+  self.d1 := (decimal4_2) L.d1;
+  self.d2 := (udecimal4_2) L.d2;
+  self.set1 := (set of string2) L.set1;
+  self.s1 := (string8) L.s1;
+  self.v1 := (varstring) L.v1;
+  self.v2 := (varstring5) L.v2;
+  self.u1 := (unicode8) L.u1;
+  self.vu1 := (varunicode) L.vu1;
+  self.vu2 := (varunicode5) L.vu2;
+  self.da1 := (data) L.da1;
+  self.da2 := (data8) L.da2;
+  self := [];
+END;
+
+
+name := dataset([{'Typecast ' + #TEXT(t)}], %dest_typecast_from_string%);
+pname := PROJECT(nofold(name), %transform_typecast_from_string%(LEFT));
+ENDMACRO;
+
+declare_typecast_from_int(integer4, typecast_from_int4, project_from_int4);
+declare_typecast_from_int(integer2, typecast_from_int2, project_from_int2);
+declare_typecast_from_int(real4, typecast_from_real4, project_from_real4);
+declare_typecast_from_int(real8, typecast_from_real8, project_from_real8);
+declare_typecast_from_int(packed integer, typecast_from_packed, project_from_packed);
+declare_typecast_from_int(packed unsigned, typecast_from_upacked, project_from_upacked);
+declare_typecast_from_int(big_endian unsigned, typecast_from_bigendian, project_from_bigendian);
+declare_typecast_from_int(decimal4_2, typecast_from_decimal, project_from_decimal);
+declare_typecast_from_int(unsigned decimal4_2, typecast_from_udecimal, project_from_udecimal);
+
+declare_typecast_from_string(string, typecast_from_string, project_from_string);
+declare_typecast_from_string(varstring, typecast_from_varstring, project_from_varstring);
+declare_typecast_from_string(string8, typecast_from_string8, project_from_string8);
+declare_typecast_from_string(varstring8, typecast_from_varstring8, project_from_varstring8);
+declare_typecast_from_string(unicode, typecast_from_unicode, project_from_unicode);
+declare_typecast_from_string(varunicode, typecast_from_varunicode, project_from_varunicode);
+declare_typecast_from_string(unicode8, typecast_from_unicode8, project_from_unicode8);
+declare_typecast_from_string(varunicode8, typecast_from_varunicode8, project_from_varunicode8);
+declare_typecast_from_string(utf8, typecast_from_utf8, project_from_utf8);
+
+sequential(
+  OUTPUT(s.stransform(untranslated)),
+  OUTPUT(s.stransform(none_needed)),
+  OUTPUT(s.stransform(removed)),
+  OUTPUT(s.stransform(swapped_only)),
+  OUTPUT(s.rtransform(switch_linkcounts)),
+  OUTPUT(s.rtransform(translate_link)),
+  OUTPUT(s.rtransform(translate_nested)),
+  OUTPUT(s.stransform(translate_set)),
+  OUTPUT(s.stransform(nomatch)),
+  OUTPUT(s.stransform(multi_adjacent)),
+  OUTPUT(s.stransform(truncate_strings)),
+  OUTPUT(s.stransform(extend_strings)),
+  OUTPUT(s.stransform(typecast_from_int4)),
+  OUTPUT(project_from_int4),
+  OUTPUT(s.stransform(typecast_from_int2)),
+  OUTPUT(project_from_int2),
+  OUTPUT(s.stransform(typecast_from_real4)),
+  OUTPUT(project_from_real4),
+  OUTPUT(s.stransform(typecast_from_real8)),
+  OUTPUT(project_from_real8),
+  OUTPUT(s.stransform(typecast_from_packed)),
+//  OUTPUT(project_from_packed),  // internal compiler error
+
+  OUTPUT(s.stransform(typecast_from_upacked)),
+//  OUTPUT(project_from_upacked),
+  OUTPUT(s.stransform(typecast_from_bigendian)),
+  OUTPUT(project_from_bigendian),
+  OUTPUT(s.stransform(typecast_from_decimal)),
+  OUTPUT(project_from_decimal),
+  OUTPUT(s.stransform(typecast_from_udecimal)),
+  OUTPUT(project_from_udecimal),
+  OUTPUT(s.stransform(typecast_from_string)),
+  OUTPUT(project_from_string),
+  OUTPUT(s.stransform(typecast_from_varstring)),
+  OUTPUT(project_from_varstring),
+  OUTPUT(s.stransform(typecast_from_string8)),
+  OUTPUT(project_from_string8),
+  OUTPUT(s.stransform(typecast_from_varstring8)),
+  OUTPUT(project_from_varstring8),
+  OUTPUT(s.stransform(typecast_from_unicode)),
+  OUTPUT(project_from_unicode),
+  OUTPUT(s.stransform(typecast_from_varunicode)),
+  OUTPUT(project_from_varunicode),
+  OUTPUT(s.stransform(typecast_from_unicode8)),
+  OUTPUT(project_from_unicode8),
+  OUTPUT(s.stransform(typecast_from_varunicode8)),
+  OUTPUT(project_from_varunicode8),
+  OUTPUT(s.stransform(typecast_from_utf8)),
+  OUTPUT(project_from_utf8),
+);