HPCC-17513 Allow field type information to be serialized/deserialized

Also contains fix for HPCC-17968 Internal errors specifying default values

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/CMakeLists.txt

@@ -42,6 +42,7 @@ include_directories (
          ./../../dali/base
          ./../../system/mp
          ./../../rtl/include 
+         ./../../rtl/eclrtl 
          ./../../system/jlib 
          ./../../common/remote
          ./../../system/security/shared

ecl/hql/hqlatoms.cpp

@@ -97,6 +97,7 @@ IAtom * balancedAtom;
 IAtom * bcdAtom;
 IAtom * beforeAtom;
 IAtom * bestAtom;
+IAtom * bitfieldOffsetAtom;
 IAtom * bitmapAtom;
 IAtom * blobAtom;
 IAtom * cAtom;
@@ -225,6 +226,7 @@ IAtom * _internal_Atom;
 IAtom * internalFlagsAtom;
 IAtom * _isFunctional_Atom;
 IAtom * _isBlobInIndex_Atom;
+IAtom * isLastBitfieldAtom;
 IAtom * isNullAtom;
 IAtom * isValidAtom;
 IAtom * jobAtom;
@@ -552,6 +554,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(bcd);
     MAKEATOM(before);
     MAKEATOM(best);
+    MAKEATOM(bitfieldOffset);
     MAKEATOM(bitmap);
     MAKEATOM(blob);
     MAKEATOM(c);
@@ -681,6 +684,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(internalFlags);
     MAKESYSATOM(isBlobInIndex);
     MAKESYSATOM(isFunctional);
+    MAKEATOM(isLastBitfield);
     MAKEATOM(isNull);
     MAKEATOM(isValid);
     MAKEATOM(job);

ecl/hql/hqlatoms.hpp

@@ -101,6 +101,7 @@ extern HQL_API IAtom * balancedAtom;
 extern HQL_API IAtom * bcdAtom;
 extern HQL_API IAtom * beforeAtom;
 extern HQL_API IAtom * bestAtom;
+extern HQL_API IAtom * bitfieldOffsetAtom;
 extern HQL_API IAtom * bitmapAtom;
 extern HQL_API IAtom * blobAtom;
 extern HQL_API IAtom * cAtom;
@@ -229,6 +230,7 @@ extern HQL_API IAtom * _internal_Atom;
 extern HQL_API IAtom * internalFlagsAtom;
 extern HQL_API IAtom * _isBlobInIndex_Atom;
 extern HQL_API IAtom * _isFunctional_Atom;
+extern HQL_API IAtom * isLastBitfieldAtom;
 extern HQL_API IAtom * isNullAtom;
 extern HQL_API IAtom * isValidAtom;
 extern HQL_API IAtom * jobAtom;

ecl/hql/hqlfold.cpp

@@ -596,8 +596,13 @@ bool checkExternFoldable(IHqlExpression* expr, unsigned foldOptions, StringBuffe
     unsigned numParam = expr->numChildren();
     for(unsigned iparam = 0; iparam < numParam; iparam++)
     {
-        if (!expr->queryChild(iparam)->queryValue())            //NB: Already folded...
+        switch (expr->queryChild(iparam)->getOperator())
+        {
+        case no_constant: case no_null: case no_all:  // NOTE: no_all still needs work elsewhere before it will be supported fully
+            break;
+        default:
             return false;
+        }
     }
 
     IHqlExpression * formals = funcdef->queryChild(1);
@@ -725,7 +730,7 @@ IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplat
 
     // create a FuncCallStack to generate a stack used to pass parameters to 
     // the called function
-    FuncCallStack fstack;
+    FuncCallStack fstack(getBoolAttribute(body, passParameterMetaAtom, false), DEFAULTSTACKSIZE);
     
     if(body->hasAttribute(templateAtom))
         fstack.pushPtr(templateContext);
@@ -824,8 +829,7 @@ IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplat
             free(tgt);
             return NULL;
         }
-        IValue * paramValue = curParam->queryValue();
-        if (fstack.push(argType, paramValue) == -1)
+        if (fstack.push(argType, curParam) == -1)
         {
             free(tgt);
             return NULL;

ecl/hql/hqlir.cpp

@@ -1422,6 +1422,11 @@ protected:
         case type_any:
             line.append(irText);
             return;
+        case type_packedint:
+            if (!info.isSigned)
+                line.append("u");
+            line.append(irText);
+            return;
         case type_int:
         case type_swapint:
             {
@@ -1439,7 +1444,6 @@ protected:
                 line.append(info.length);
             return;
         case type_decimal:
-        case type_packedint:
         case type_bitfield:
         case type_enumerated:
             return;
@@ -1521,6 +1525,7 @@ protected:
             break;
         case type_int:
         case type_swapint:
+        case type_packedint:
             {
                 if (true)//info.isSigned)
                     line.append((__int64)info.intValue);
@@ -1534,7 +1539,6 @@ protected:
                 break;
             }
         case type_decimal:
-        case type_packedint:
         case type_string:
         case type_bitfield:
         case type_enumerated:
@@ -2063,13 +2067,13 @@ id_t ExpressionIRPlayer::doProcessConstant(IHqlExpression * expr)
         break;
     case type_int:
     case type_swapint:
+    case type_packedint:
         info.intValue = value->getIntValue();
         break;
     case type_real:
         info.realValue = value->getRealValue();
         break;
     case type_decimal:
-    case type_packedint:
     case type_string:
     case type_bitfield:
     case type_enumerated:

ecl/hql/hqlstack.cpp

@@ -16,9 +16,67 @@
 ############################################################################## */
 #include <stdlib.h>
 #include <string.h>
+#include "jlib.hpp"
+#include "eclhelper.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
+#include "rtlfield.hpp"
+#include "rtlds_imp.hpp"
+namespace hqlstack {
+#include "eclhelper_base.hpp"
+}
+#include "rtlrecord.hpp"
+#include "rtldynfield.hpp"
 #include "hqlstack.hpp"
+#include "hqlir.hpp"
+#include "hqlutil.hpp"
+
+/**
+ * class CDynamicOutputMetaData
+ *
+ * An implementation of IOutputMetaData for use with a dynamically-created record type info structure
+ *
+ */
+
+class CDynamicOutputMetaData : public hqlstack::COutputMetaData
+{
+public:
+    CDynamicOutputMetaData(const RtlRecordTypeInfo & fields)
+    : offsetInformation(fields, true), typeInfo(fields)
+    {
 
-FuncCallStack::FuncCallStack(int size) {
+    }
+
+    virtual const RtlTypeInfo * queryTypeInfo() const { return &typeInfo; }
+    virtual size32_t getRecordSize(const void * row)
+    {
+        //Allocate a temporary offset array on the stack to avoid runtime overhead.
+        unsigned numOffsets = offsetInformation.getNumVarFields() + 1;
+        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow offsetCalculator(offsetInformation, row, numOffsets, variableOffsets);
+        return offsetCalculator.getRecordSize();
+    }
+
+    virtual size32_t getFixedSize() const
+    {
+        return offsetInformation.getFixedSize();
+    }
+    // returns 0 for variable row size
+    virtual size32_t getMinRecordSize() const
+    {
+        return offsetInformation.getMinRecordSize();
+    }
+
+    virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { throwUnexpected(); }
+
+protected:
+    RtlRecord offsetInformation;
+    const RtlTypeInfo &typeInfo;
+};
+
+FuncCallStack::FuncCallStack(bool _hasMeta, int size)
+{
+    hasMeta = _hasMeta;
     if(size < DEFAULTSTACKSIZE)
         size = DEFAULTSTACKSIZE;
     sp = 0;
@@ -81,17 +139,23 @@ int FuncCallStack::push(unsigned len, const void * data)
 
 
 
-int FuncCallStack::push(ITypeInfo* argType, IValue* paramValue) 
+int FuncCallStack::push(ITypeInfo* argType, IHqlExpression* curParam)
 {
     unsigned len = 0;
     char* str;
     int incsize;
     int inclen;
 
-    Owned<IValue> castParam = paramValue->castTo(argType);
-    if(!castParam) {
-        PrintLog("Failed to cast paramValue to argType in FuncCallStack::push");
-        return -1;
+    IValue * paramValue = curParam->queryValue();
+    Owned<IValue> castParam;
+    if (paramValue) // Not all constants have a paramValue - null, all, constant records etc
+    {
+        castParam.setown(paramValue->castTo(argType));
+        if(!castParam)
+        {
+            PrintLog("Failed to cast paramValue to argType in FuncCallStack::push");
+            return -1;
+        }
     }
 
     switch (argType->getTypeCode()) 
@@ -178,7 +242,36 @@ int FuncCallStack::push(ITypeInfo* argType, IValue* paramValue)
         memset(stackbuf+sp+incsize, 0, inclen - incsize);
         sp += inclen;
         break;
+    case type_row:
+    {
+        if (hasMeta)
+        {
+            try
+            {
+                pushMeta(curParam->queryRecordType());
+            }
+            catch (IException *E)
+            {
+                ::Release(E);
+                return -1;
+            }
+        }
+        if (curParam->getOperator()==no_null)
+        {
+            // MORE - check type matches
+            MemoryBuffer out;
+            createConstantNullRow(out, curParam->queryRecord());
+            str = (char *) out.detach();
+            push(sizeof(char *), &str);
+            if(numToFree < MAXARGS)
+                toFree[numToFree++] = str;
+        }
+        else
+            return -1;
+        break;
+    }
     default:
+        EclIR::dump_ir(curParam);
         //code isn't here to pass sets/datasets to external functions....
         return -1;
     }
@@ -186,6 +279,16 @@ int FuncCallStack::push(ITypeInfo* argType, IValue* paramValue)
     return sp;
 }
 
+int FuncCallStack::pushMeta(ITypeInfo *type)
+{
+    if (!deserializer)
+        deserializer.setown(createRtlFieldTypeDeserializer());
+    const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), type);
+    CDynamicOutputMetaData * meta = new CDynamicOutputMetaData(* static_cast<const RtlRecordTypeInfo *>(typeInfo));
+    metas.append(*meta);
+    return pushPtr(meta);
+}
+
 int FuncCallStack::pushPtr(void * val)
 {
     return push(sizeof(void *), &val);

ecl/hql/hqlstack.hpp

@@ -21,6 +21,7 @@
 #include "jstream.hpp"
 #include "jexcept.hpp"
 #include "jmisc.hpp"
+#include "rtldynfield.hpp"
 #include "deftype.hpp"
 #include "defvalue.hpp"
 #include "hqlexpr.hpp"
@@ -88,6 +89,9 @@ private:
     char*      stackbuf;
     char*      toFree[MAXARGS];
     int        numToFree;
+    bool       hasMeta;
+    Owned<IRtlFieldTypeDeserializer> deserializer;
+    IArrayOf<IOutputMetaData> metas;
 #ifdef MAXFPREGS
  #ifdef FPREG_FIXEDSIZE
     double      fpRegs[MAXFPREGS];
@@ -102,7 +106,7 @@ private:
 #endif
     unsigned align(unsigned size);
 public:
-    FuncCallStack(int size = DEFAULTSTACKSIZE);
+    FuncCallStack(bool hasMeta, int size);
     virtual ~FuncCallStack();
 
     unsigned getSp();
@@ -114,7 +118,8 @@ public:
  #endif
 #endif
 
-    int push(ITypeInfo* argType, IValue* paramValue);
+    int push(ITypeInfo* argType, IHqlExpression* curParam);
+    int pushMeta(ITypeInfo *type);
     int push(char* & val);
     int pushPtr(void * val);
     int pushRef(unsigned& val);

ecl/hql/hqlutil.cpp

@@ -17,6 +17,7 @@
 #include "jliball.hpp"
 #include "hql.hpp"
 #include "eclrtl.hpp"
+#include "rtldynfield.hpp"
 
 #include "platform.h"
 #include "jlib.hpp"
@@ -7701,6 +7702,7 @@ public:
 
         if (formals->numChildren())
         {
+            bool hasMeta = getBoolAttribute(body, passParameterMetaAtom, false);
             ForEachChild(i, formals)
             {
                 IHqlExpression * param = formals->queryChild(i);
@@ -7711,7 +7713,7 @@ public:
 
                 if (isOut)
                     mangled.append("R");
-                if (!mangleSimpleType(mangled, paramType, isConst))
+                if (!mangleSimpleType(mangled, paramType, isConst, hasMeta))
                     return false;
             }
         }
@@ -7721,7 +7723,7 @@ public:
     }
 
 protected:
-    bool mangleSimpleType(StringBuffer & result, ITypeInfo * type, bool hasConst)
+    bool mangleSimpleType(StringBuffer & result, ITypeInfo * type, bool hasConst, bool hasMeta)
     {
         if (!type)
             return false;
@@ -7768,13 +7770,13 @@ protected:
             result.append("c");
             return true;
         case type_enumerated:
-            return mangleSimpleType(result, type->queryChildType(), hasConst);
+            return mangleSimpleType(result, type->queryChildType(), hasConst, hasMeta);
         case type_pointer:
             result.append("P");
-            return mangleSimpleType(result, type->queryChildType(), hasConst);
+            return mangleSimpleType(result, type->queryChildType(), hasConst, hasMeta);
         case type_array:
             result.append("A").append(type->getSize()).append("_");;
-            return mangleSimpleType(result, type->queryChildType(), hasConst);
+            return mangleSimpleType(result, type->queryChildType(), hasConst, hasMeta);
         case type_table:
         case type_groupedtable:
             result.append("j"); // size32_t
@@ -7786,7 +7788,9 @@ protected:
             result.append(lookupRepeat(hasConst ? "PKv" : "Pv")); // *
             return true;
         case type_row:
-            result.append(lookupRepeat("Ph"));
+            if (hasMeta)
+                result.append(lookupRepeat("R15IOutputMetaData"));
+            result.append(lookupRepeat("PKh"));  // Does not seem to depend on const
             return true;
         case type_void:
             result.append("v");
@@ -8684,10 +8688,16 @@ bool ConstantRowCreator::processFieldValue(IHqlExpression * optLhs, ITypeInfo *
     switch (lhsType->getTypeCode())
     {
     case type_packedint:
+    {
         if (!rhs->queryValue())
             return false;
-        //MORE: Could handle this...
-        return false;
+        unsigned orig = out.length();
+        void *tgt = out.reserve(9);
+        rtlSetPackedUnsigned(tgt, rhs->queryValue()->getIntValue());
+        unsigned actualSize = rtlGetPackedSize(tgt);
+        out.setLength(orig+actualSize);
+        return true;
+    }
     case type_set:
         if (isNullList(rhs))
         {
@@ -8809,6 +8819,22 @@ bool ConstantRowCreator::processFieldValue(IHqlExpression * optLhs, ITypeInfo *
             castValue->toMem(out.reserve(lenValue));
             return true;
         }
+    case type_varunicode:
+    {
+        if (sizeLhs == UNKNOWN_LENGTH)
+        {
+            void * target = out.reserve((lenValue+1)*sizeof(UChar));
+            castValue->toMem(target);
+        }
+        else
+        {
+            UChar * target = (UChar *) out.reserve(sizeLhs*sizeof(UChar));
+            for (size32_t pos = 0; pos < sizeLhs; pos++)
+                target[pos] = (UChar) ' ';
+            castValue->toMem(target);
+        }
+        return true;
+    }
     case type_varstring:
         {
             //Move to else
@@ -8819,9 +8845,6 @@ bool ConstantRowCreator::processFieldValue(IHqlExpression * optLhs, ITypeInfo *
             }
             else
             {
-                //Disabled for the moment to prevent the size of generated expressions getting too big.
-                if (sizeLhs > 40)
-                    return false;
                 void * target = out.reserve(sizeLhs);
                 memset(target, ' ', sizeLhs);   // spaces expand better in the c++
                 castValue->toMem(target);
@@ -8837,9 +8860,6 @@ bool ConstantRowCreator::processFieldValue(IHqlExpression * optLhs, ITypeInfo *
             castValue->toMem(out.reserve(castValue->getSize()));
             return true;
         }
-    //MORE:
-    //type_varunicode
-    //type_packedint
     }
     return false;
 }
@@ -9493,3 +9513,337 @@ IHqlExpression * convertSetToExpression(bool isAll, size32_t len, const void * p
     }
     return createValue(no_list, LINK(setType), results);
 }
+
+//-------------------------------------------------------------------------------------------------
+
+void getFieldTypeInfo(FieldTypeInfoStruct &out, ITypeInfo *type)
+{
+    assertex(type);
+    type_t tc = type->getTypeCode();
+    if (tc == type_record)
+        type = queryUnqualifiedType(type);
+
+    if (tc == type_alien)
+    {
+        ITypeInfo * physicalType = queryAlienType(type)->queryPhysicalType();
+        if (physicalType->getSize() != UNKNOWN_LENGTH)
+        {
+            //Don't use the generated class for xml generation since it will generate physical rather than logical
+            out.fieldType |= (RFTMalien|RFTMinvalidxml|RFTMnoserialize);
+            type = physicalType;
+            tc = type->getTypeCode();
+        }
+        else
+        {
+            out.fieldType |= RFTMunknownsize;
+            //can't work out the size of the field - to keep it as unknown for the moment.
+            //until the alien field type is supported
+        }
+    }
+    out.fieldType |= tc;
+    out.length = type->getSize();
+    out.locale = nullptr;
+    out.className = nullptr;
+    if (out.length == UNKNOWN_LENGTH)
+    {
+        out.fieldType |= RFTMunknownsize;
+        out.length = 0;
+    }
+
+    switch (tc)
+    {
+    case type_boolean:
+        out.className = "RtlBoolTypeInfo";
+        break;
+    case type_real:
+        out.className ="RtlRealTypeInfo";
+        break;
+    case type_date:
+    case type_enumerated:
+    case type_int:
+        out.className = "RtlIntTypeInfo";
+        if (!type->isSigned())
+            out.fieldType |= RFTMunsigned;
+        break;
+    case type_swapint:
+        out.className = "RtlSwapIntTypeInfo";
+        if (!type->isSigned())
+            out.fieldType |= RFTMunsigned;
+        break;
+    case type_packedint:
+        out.className = "RtlPackedIntTypeInfo";
+        if (!type->isSigned())
+            out.fieldType |= RFTMunsigned;
+        break;
+    case type_decimal:
+        out.className = "RtlDecimalTypeInfo";
+        if (!type->isSigned())
+            out.fieldType |= RFTMunsigned;
+        out.length = type->getDigits() | (type->getPrecision() << 16);
+        break;
+    case type_char:
+        out.className = "RtlCharTypeInfo";
+        break;
+    case type_data:
+        out.className = "RtlDataTypeInfo";
+        break;
+    case type_qstring:
+        out.className = "RtlQStringTypeInfo";
+        out.length = type->getStringLen();
+        break;
+    case type_varstring:
+        out.className = "RtlVarStringTypeInfo";
+        if (type->queryCharset() && type->queryCharset()->queryName()==ebcdicAtom)
+            out.fieldType |= RFTMebcdic;
+        out.length = type->getStringLen();
+        break;
+    case type_string:
+        out.className = "RtlStringTypeInfo";
+        if (type->queryCharset() && type->queryCharset()->queryName()==ebcdicAtom)
+            out.fieldType |= RFTMebcdic;
+        break;
+    case type_bitfield:
+        {
+        out.className = "RtlBitfieldTypeInfo";
+        unsigned size = type->queryChildType()->getSize();
+        unsigned bitsize = type->getBitSize();
+        unsigned offset = (unsigned)getIntValue(queryAttributeChild(type, bitfieldOffsetAtom, 0),-1);
+        bool isLastBitfield = (queryAttribute(type, isLastBitfieldAtom) != NULL);
+        if (isLastBitfield)
+            out.fieldType |= RFTMislastbitfield;
+        if (!type->isSigned())
+            out.fieldType |= RFTMunsigned;
+        out.length = size | (bitsize << 8) | (offset << 16);
+        break;
+        }
+    case type_record:
+        {
+            IHqlExpression * record = ::queryRecord(type);
+            out.className = "RtlRecordTypeInfo";
+            out.length = getMinRecordSize(record);
+            if (!isFixedSizeRecord(record))
+                out.fieldType |= RFTMunknownsize;
+            break;
+        }
+    case type_row:
+        {
+            out.className = "RtlRowTypeInfo";
+            if (hasLinkCountedModifier(type))
+                out.fieldType |= RFTMlinkcounted;
+            break;
+        }
+    case type_table:
+    case type_groupedtable:
+        {
+            out.className = "RtlDatasetTypeInfo";
+            if (hasLinkCountedModifier(type))
+            {
+                out.fieldType |= RFTMlinkcounted;
+                out.fieldType &= ~RFTMunknownsize;
+            }
+            break;
+        }
+    case type_dictionary:
+        {
+            out.className = "RtlDictionaryTypeInfo";
+            out.fieldType |= RFTMnoserialize;
+            if (hasLinkCountedModifier(type))
+            {
+                out.fieldType |= RFTMlinkcounted;
+                out.fieldType &= ~RFTMunknownsize;
+            }
+            break;
+        }
+    case type_set:
+        out.className = "RtlSetTypeInfo";
+        break;
+    case type_unicode:
+        out.className = "RtlUnicodeTypeInfo";
+        out.locale = str(type->queryLocale());
+        out.length = type->getStringLen();
+        break;
+    case type_varunicode:
+        out.className = "RtlVarUnicodeTypeInfo";
+        out.locale = str(type->queryLocale());
+        out.length = type->getStringLen();
+        break;
+    case type_utf8:
+        out.className = "RtlUtf8TypeInfo";
+        out.locale = str(type->queryLocale());
+        out.length = type->getStringLen();
+        break;
+    case type_blob:
+    case type_pointer:
+    case type_class:
+    case type_array:
+    case type_void:
+    case type_alien:
+    case type_none:
+    case type_any:
+    case type_pattern:
+    case type_rule:
+    case type_token:
+    case type_feature:
+    case type_event:
+    case type_null:
+    case type_scope:
+    case type_transform:
+    default:
+        out.className = "RtlUnimplementedTypeInfo";
+        out.fieldType |= (RFTMcontainsunknown|RFTMinvalidxml|RFTMnoserialize);
+        break;
+    }
+}
+
+bool checkXpathIsNonScalar(const char *xpath)
+{
+    return (strpbrk(xpath, "/?*[]<>")!=NULL); //anything other than a single tag/attr name cannot name a scalar field
+}
+
+unsigned buildRtlRecordFields(IRtlFieldTypeDeserializer &deserializer, unsigned &idx, const RtlFieldInfo * * fieldsArray, IHqlExpression *record, IHqlExpression *rowRecord)
+{
+    unsigned typeFlags = 0;
+    ForEachChild(i, record)
+    {
+        unsigned fieldFlags = 0;
+        IHqlExpression * field = record->queryChild(i);
+        switch (field->getOperator())
+        {
+        case no_ifblock:
+            typeFlags |= RFTMnoserialize;
+            break;
+        case no_field:
+        {
+            ITypeInfo *fieldType = field->queryType();
+            switch (fieldType->getTypeCode())
+            {
+            case type_alien:
+                //MORE:::
+                break;
+            case type_row:
+                //Backward compatibility - should revisit
+                fieldType = fieldType->queryChildType();
+                break;
+            case type_bitfield:
+                UNIMPLEMENTED;
+                break;
+            }
+
+            const RtlTypeInfo *type = buildRtlType(deserializer, fieldType);
+            typeFlags |= type->fieldType & RFTMinherited;
+            StringBuffer lowerName;
+            lowerName.append(field->queryName()).toLowerCase();
+
+            StringBuffer xpathName, xpathItem;
+            switch (field->queryType()->getTypeCode())
+            {
+            case type_set:
+                extractXmlName(xpathName, &xpathItem, NULL, field, "Item", false);
+                break;
+            case type_dictionary:
+            case type_table:
+            case type_groupedtable:
+                extractXmlName(xpathName, &xpathItem, NULL, field, "Row", false);
+                //Following should be in the type processing, and the type should include the information
+                if (field->hasAttribute(sizeAtom) || field->hasAttribute(countAtom))
+                    fieldFlags |= RFTMinvalidxml;
+                break;
+            default:
+                extractXmlName(xpathName, NULL, NULL, field, NULL, false);
+                break;
+            }
+            //Format of the xpath field is (nested-item 0x01 repeated-item)
+            if (xpathItem.length())
+                xpathName.append(xpathCompoundSeparatorChar).append(xpathItem);
+            if (xpathName.charAt(0) == '@')
+                fieldFlags |= RFTMhasxmlattr;
+            if (checkXpathIsNonScalar(xpathName))
+                fieldFlags |= RFTMhasnonscalarxpath;
+            const char *xpath = xpathName.str();
+            if (strcmp(lowerName, xpath)==0)
+                xpath = nullptr;
+
+            MemoryBuffer defaultInitializer;
+            IHqlExpression *defaultValue = queryAttributeChild(field, defaultAtom, 0);
+            if (defaultValue)
+            {
+                LinkedHqlExpr targetField = field;
+                if (fieldType->getTypeCode() == type_bitfield)
+                    targetField.setown(createField(field->queryId(), LINK(fieldType->queryChildType()), NULL));
+
+                if (!createConstantField(defaultInitializer, targetField, defaultValue))
+                    UNIMPLEMENTED;  // MORE - fail more gracefully!
+            }
+            fieldsArray[idx] = deserializer.addFieldInfo(lowerName, xpath, type, fieldFlags, (const char *) defaultInitializer.detach());
+            typeFlags |= fieldFlags & RFTMinherited;
+            idx++;
+            break;
+        }
+        case no_record:
+            typeFlags |= buildRtlRecordFields(deserializer, idx, fieldsArray, field, rowRecord);
+            break;
+        }
+    }
+    return typeFlags;
+}
+
+const RtlTypeInfo *buildRtlType(IRtlFieldTypeDeserializer &deserializer, ITypeInfo *type)
+{
+    assertex(type);
+    switch (type->getTypeCode())
+    {
+    case type_alien:
+        //MORE:::
+        break;
+    case type_row:
+        //Backward compatibility - should revisit
+        return buildRtlType(deserializer, type->queryChildType());
+    //case type_bitfield:
+        //fieldKey contains a field with a type annotated with offsets/isLastBitfield
+        //OwnedHqlExpr fieldKey = getRtlFieldKey(field, rowRecord);
+        //return buildRtlType(deserializer, fieldKey->queryType());
+    }
+
+    const RtlTypeInfo * found = deserializer.lookupType(type);
+    if (found)
+        return found;
+
+    FieldTypeInfoStruct info;
+    getFieldTypeInfo(info, type);
+
+    switch (info.fieldType & RFTMkind)
+    {
+    case type_record:
+        {
+            IHqlExpression * record = ::queryRecord(type);
+            unsigned numFields = getFlatFieldCount(record);
+            info.fieldsArray = new const RtlFieldInfo * [numFields+1];
+            unsigned idx = 0;
+            info.fieldType |= buildRtlRecordFields(deserializer, idx, info.fieldsArray, record, record);
+            info.fieldsArray[idx] = nullptr;
+            break;
+        }
+    case type_row:
+        {
+            info.childType = buildRtlType(deserializer, ::queryRecordType(type));
+            break;
+        }
+    case type_table:
+    case type_groupedtable:
+        {
+            info.childType = buildRtlType(deserializer, ::queryRecordType(type));
+            break;
+        }
+    case type_dictionary:
+        return nullptr;  // MORE - does this leak?
+    case type_set:
+        info.childType = buildRtlType(deserializer, type->queryChildType());
+        break;
+    }
+    if (info.childType)
+        info.fieldType |= info.childType->fieldType & RFTMinherited;
+
+    return deserializer.addType(info, type);
+}
+
+

ecl/hql/hqlutil.hpp

@@ -742,6 +742,35 @@ extern HQL_API IHqlExpression * queryTransformAssign(IHqlExpression * transform,
 extern HQL_API IHqlExpression * queryTransformAssignValue(IHqlExpression * transform, IHqlExpression * searchField);
 extern HQL_API IHqlExpression * convertSetToExpression(bool isAll, size32_t len, const void * ptr, ITypeInfo * setType);
 
+struct FieldTypeInfoStruct;
+interface IRtlFieldTypeDeserializer;
+class RtlTypeInfo;
+
+/*
+ * Check whether an xpath contains any non-scalar elements (and is this unsuitable for use when generating ECL)
+ *
+ * @param  xpath The xpath to check
+ * @return True if non-scalar elements are present.
+ */
+extern HQL_API bool checkXpathIsNonScalar(const char *xpath);
+
+/*
+ * Fill in field type information for specified type, for creation of runtime type information from compiler structures
+ *
+ * @param out  Filled in with resulting information
+ * @param type Compiler type
+ */
+extern HQL_API void getFieldTypeInfo(FieldTypeInfoStruct &out, ITypeInfo *type);
+
+/*
+ * Build a runtime type information structure from a compile-time type descriptor
+ *
+ * @param  deserializer Deserializer object used to create and own the resulting types
+ * @param  type         Compiler type information structure
+ * @return              Run-time type information structure, owned by supplied deserializer
+ */
+extern HQL_API const RtlTypeInfo *buildRtlType(IRtlFieldTypeDeserializer &deserializer, ITypeInfo *type);
+
 extern HQL_API bool isCommonSubstringRange(IHqlExpression * expr);
 extern HQL_API bool isFileOutput(IHqlExpression * expr);
 extern HQL_API bool isWorkunitOutput(IHqlExpression * expr);

ecl/hqlcpp/hqlcatom.cpp

@@ -34,7 +34,6 @@ IAtom * activeMatchUtf8Atom;
 IAtom * activeProductionMarkerAtom;
 IAtom * activeValidateMarkerAtom;
 IAtom * activityIdMarkerAtom;
-IAtom * bitfieldOffsetAtom;
 IAtom * blobHelperAtom;
 IAtom * branchAtom;
 IAtom * checkpointAtom;
@@ -64,7 +63,6 @@ IAtom * initAtom;
 IAtom * insideOnCreateAtom;
 IAtom * insideOnStartAtom;
 IAtom * instanceAtom;
-IAtom * isLastBitfieldAtom;
 IAtom * _loop_Atom;
 IAtom * _loopFirst_Atom;
 IAtom * mainprototypesAtom;
@@ -1438,7 +1436,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(activeProductionMarker);
     MAKEATOM(activeValidateMarker);
     MAKEATOM(activityIdMarker);
-    MAKEATOM(bitfieldOffset);
     MAKEATOM(blobHelper);
     MAKEATOM(branch);
     MAKEATOM(checkpoint);
@@ -1467,7 +1464,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(insideOnCreate);
     MAKEATOM(insideOnStart);
     MAKEATOM(instance);
-    MAKEATOM(isLastBitfield);
     MAKEATOM(mainprototypes);
     MAKEATOM(multiInstance);
     MAKEATOM(noSet);

ecl/hqlcpp/hqlcatom.hpp

@@ -32,7 +32,6 @@ extern IAtom * activeMatchUtf8Atom;
 extern IAtom * activeProductionMarkerAtom;
 extern IAtom * activeValidateMarkerAtom;
 extern IAtom * activityIdMarkerAtom;
-extern IAtom * bitfieldOffsetAtom;
 extern IAtom * blobHelperAtom;
 extern IAtom * branchAtom;
 extern IAtom * checkpointAtom;
@@ -62,7 +61,6 @@ extern IAtom * initAtom;
 extern IAtom * insideOnCreateAtom;
 extern IAtom * insideOnStartAtom;
 extern IAtom * instanceAtom;
-extern IAtom * isLastBitfieldAtom;
 extern IAtom * _loop_Atom;
 extern IAtom * _loopFirst_Atom;
 extern IAtom * mainprototypesAtom;

ecl/hqlcpp/hqlcpp.ipp

@@ -1056,7 +1056,7 @@ public:
 
     IHqlExpression * getRtlFieldKey(IHqlExpression * expr, IHqlExpression * ownerRecord);
     unsigned buildRtlField(StringBuffer & instanceName, IHqlExpression * field, IHqlExpression * rowRecord);
-    unsigned buildRtlType(StringBuffer & instanceName, ITypeInfo * type, unsigned typeFlags);
+    unsigned buildRtlType(StringBuffer & instanceName, ITypeInfo * type);
     unsigned buildRtlRecordFields(StringBuffer & instanceName, IHqlExpression * record, IHqlExpression * rowRecord);
     unsigned expandRtlRecordFields(StringBuffer & fieldListText, IHqlExpression * record, IHqlExpression * rowRecord);
     unsigned buildRtlIfBlockField(StringBuffer & instanceName, IHqlExpression * ifblock, IHqlExpression * rowRecord);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -22,6 +22,7 @@
 #include "jmisc.hpp"
 #include "jstream.ipp"
 #include "jdebug.hpp"
+#include "rtldynfield.hpp"
 
 #include "build-config.h"
 #include "hql.hpp"
@@ -3620,11 +3621,6 @@ IHqlExpression * HqlCppTranslator::getRtlFieldKey(IHqlExpression * expr, IHqlExp
     return LINK(expr);
 }
 
-bool checkXpathIsNonScalar(const char *xpath)
-{
-    return (strpbrk(xpath, "/?*[]<>")!=NULL); //anything other than a single tag/attr name cannot name a scalar field
-}
-
 unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpression * field, IHqlExpression * rowRecord)
 {
     OwnedHqlExpr fieldKey = getRtlFieldKey(field, rowRecord);
@@ -3640,6 +3636,7 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpres
 
     StringBuffer name;
     unsigned typeFlags = 0;
+    unsigned fieldFlags = 0;
     if (field->getOperator() == no_ifblock)
     {
         typeFlags = buildRtlIfBlockField(name, field, rowRecord);
@@ -3674,7 +3671,7 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpres
             extractXmlName(xpathName, &xpathItem, NULL, field, "Row", false);
             //Following should be in the type processing, and the type should include the information
             if (field->hasAttribute(sizeAtom) || field->hasAttribute(countAtom))
-                typeFlags |= RFTMinvalidxml;
+                fieldFlags |= RFTMinvalidxml;
             break;
         default:
             extractXmlName(xpathName, NULL, NULL, field, NULL, false);
@@ -3684,9 +3681,9 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpres
         if (xpathName.length())
         {
             if (xpathName.charAt(0) == '@')
-                typeFlags |= RFTMhasxmlattr;
+                fieldFlags |= RFTMhasxmlattr;
             if (checkXpathIsNonScalar(xpathName))
-                typeFlags |= RFTMhasnonscalarxpath;
+                fieldFlags |= RFTMhasnonscalarxpath;
         }
 
         StringBuffer lowerName;
@@ -3729,9 +3726,12 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpres
 
         StringBuffer definition;
         StringBuffer typeName;
-        typeFlags = buildRtlType(typeName, fieldType, typeFlags); //benefit to adding other flags to generated code as well?
+        typeFlags |= buildRtlType(typeName, fieldType);
+        typeFlags |= fieldFlags;
 
         definition.append("const RtlFieldStrInfo ").append(name).append("(\"").append(lowerName).append("\",").append(xpathCppText).append(",&").append(typeName);
+        if (fieldFlags || defaultInitializer.length())
+            definition.append(',').appendf("0x%x", fieldFlags);
         if (defaultInitializer.length())
             definition.append(',').append(defaultInitializer);
         definition.append(");");
@@ -3756,12 +3756,12 @@ unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHq
     BuildCtx declarectx(*code, declareAtom);
 
     //First generate a pseudo type entry for an ifblock.
-    unsigned fieldType = type_ifblock|RFTMcontainsifblock;
+    unsigned fieldType = type_ifblock|RFTMcontainsifblock|RFTMnoserialize;
     {
         unsigned length = 0;
         StringBuffer childTypeName;
         unsigned childType = buildRtlRecordFields(childTypeName, ifblock->queryChild(1), rowRecord);
-        fieldType |= (childType & (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr));
+        fieldType |= (childType & RFTMinherited);
 
         StringBuffer className;
         typeName.append("ty").append(++nextTypeId);
@@ -3822,7 +3822,7 @@ unsigned HqlCppTranslator::expandRtlRecordFields(StringBuffer & fieldListText, I
             childType = expandRtlRecordFields(fieldListText, cur, rowRecord);
             break;
         }
-        fieldType |= (childType & (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr));
+        fieldType |= (childType & RFTMinherited);
     }
     return fieldType;
 }
@@ -3847,15 +3847,12 @@ unsigned HqlCppTranslator::buildRtlRecordFields(StringBuffer & instanceName, IHq
     return fieldFlags;
 }
 
-unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo * type, unsigned typeFlags)
+unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo * type)
 {
     assertex(type);
-    type_t tc = type->getTypeCode();
-    if (tc == type_record)
+    if (type->getTypeCode() == type_record)
         type = queryUnqualifiedType(type);
-
-    VStringBuffer searchKey("t%d", typeFlags);
-    OwnedHqlExpr search = createVariable(searchKey, LINK(type));
+    OwnedHqlExpr search = createVariable("t", LINK(type));
     BuildCtx declarectx(*code, declareAtom);
     HqlExprAssociation * match = declarectx.queryMatchExpr(search);
     if (match)
@@ -3865,7 +3862,7 @@ unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo *
         return (unsigned)getIntValue(value->queryChild(1));
     }
 
-    StringBuffer name, className, arguments;
+    StringBuffer name, arguments;
     if (options.debugGeneratedCpp)
     {
         StringBuffer ecl;
@@ -3877,103 +3874,15 @@ unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo *
     else
         name.append("ty").append(++nextTypeId);
 
-    unsigned fieldType = typeFlags;
-    if (tc == type_alien)
-    {
-        ITypeInfo * physicalType = queryAlienType(type)->queryPhysicalType();
-        if (physicalType->getSize() != UNKNOWN_LENGTH)
-        {
-            //Don't use the generated class for xml generation since it will generate physical rather than logical
-            fieldType |= (RFTMalien|RFTMinvalidxml);
-            type = physicalType;
-            tc = type->getTypeCode();
-        }
-        else
-        {
-            fieldType |= RFTMunknownsize;
-            //can't work out the size of the field - to keep it as unknown for the moment.
-            //until the alien field type is supported
-        }
-    }
-    fieldType |= tc;
-    unsigned length = type->getSize();
-    if (length == UNKNOWN_LENGTH)
-    {
-        fieldType |= RFTMunknownsize;
-        length = 0;
-    }
-
+    FieldTypeInfoStruct info;
+    getFieldTypeInfo(info, type);
     unsigned childType = 0;
-    switch (tc)
+
+    switch (info.fieldType & RFTMkind)
     {
-    case type_boolean:
-        className.append("RtlBoolTypeInfo");
-        break;
-    case type_real:
-        className.append("RtlRealTypeInfo");
-        break;
-    case type_date:
-    case type_enumerated:
-    case type_int:
-        className.append("RtlIntTypeInfo");
-        if (!type->isSigned())
-            fieldType |= RFTMunsigned;
-        break;
-    case type_swapint:
-        className.append("RtlSwapIntTypeInfo");
-        if (!type->isSigned())
-            fieldType |= RFTMunsigned;
-        break;
-    case type_packedint:
-        className.append("RtlPackedIntTypeInfo");
-        if (!type->isSigned())
-            fieldType |= RFTMunsigned;
-        break;
-    case type_decimal:
-        className.append("RtlDecimalTypeInfo");
-        if (!type->isSigned())
-            fieldType |= RFTMunsigned;
-        length = type->getDigits() | (type->getPrecision() << 16);
-        break;
-    case type_char:
-        className.append("RtlCharTypeInfo");
-        break;
-    case type_data:
-        className.append("RtlDataTypeInfo");
-        break;
-    case type_qstring:
-        className.append("RtlQStringTypeInfo");
-        length = type->getStringLen();
-        break;
-    case type_varstring:
-        className.append("RtlVarStringTypeInfo");
-        if (type->queryCharset() && type->queryCharset()->queryName()==ebcdicAtom)
-            fieldType |= RFTMebcdic;
-        length = type->getStringLen();
-        break;
-    case type_string:
-        className.append("RtlStringTypeInfo");
-        if (type->queryCharset() && type->queryCharset()->queryName()==ebcdicAtom)
-            fieldType |= RFTMebcdic;
-        break;
-    case type_bitfield:
-        {
-        className.append("RtlBitfieldTypeInfo");
-        unsigned size = type->queryChildType()->getSize();
-        unsigned bitsize = type->getBitSize();
-        unsigned offset = (unsigned)getIntValue(queryAttributeChild(type, bitfieldOffsetAtom, 0),-1);
-        bool isLastBitfield = (queryAttribute(type, isLastBitfieldAtom) != NULL);
-        if (isLastBitfield)
-            fieldType |= RFTMislastbitfield;
-        if (!type->isSigned())
-            fieldType |= RFTMunsigned;
-        length = size | (bitsize << 8) | (offset << 16);
-        break;
-        }
     case type_record:
         {
             IHqlExpression * record = ::queryRecord(type);
-            className.append("RtlRecordTypeInfo");
             arguments.append(",");
             StringBuffer fieldsInstance;
             childType = buildRtlRecordFields(fieldsInstance, record, record);
@@ -3993,106 +3902,54 @@ unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo *
                 childType |= buildRtlRecordFields(arguments, record, record, true);
             }
 #endif
-
-//          fieldType |= (childType & RFTMcontainsifblock);
-            length = getMinRecordSize(record);
-            if (!isFixedRecordSize(record))
-                fieldType |= RFTMunknownsize;
             break;
         }
     case type_row:
         {
-            className.clear().append("RtlRowTypeInfo");
             arguments.append(",&");
-            childType = buildRtlType(arguments, ::queryRecordType(type), 0);
-//          fieldType |= (childType & RFTMcontainsifblock);
-            if (hasLinkCountedModifier(type))
-                fieldType |= RFTMlinkcounted;
+            childType = buildRtlType(arguments, ::queryRecordType(type));
             break;
         }
     case type_table:
     case type_groupedtable:
         {
-            className.clear().append("RtlDatasetTypeInfo");
             arguments.append(",&");
-            childType = buildRtlType(arguments, ::queryRecordType(type), 0);
-            if (hasLinkCountedModifier(type))
-            {
-                fieldType |= RFTMlinkcounted;
-                fieldType &= ~RFTMunknownsize;
-            }
+            childType = buildRtlType(arguments, ::queryRecordType(type));
             break;
         }
     case type_dictionary:
         {
-            className.clear().append("RtlDictionaryTypeInfo");
             arguments.append(",&");
-            childType = buildRtlType(arguments, ::queryRecordType(type), 0);
-            if (hasLinkCountedModifier(type))
-            {
-                fieldType |= RFTMlinkcounted;
-                fieldType &= ~RFTMunknownsize;
-            }
+            childType = buildRtlType(arguments, ::queryRecordType(type));
             StringBuffer lookupHelperName;
             buildDictionaryHashClass(::queryRecord(type), lookupHelperName);
             arguments.append(",&").append(lookupHelperName.str());
             break;
         }
     case type_set:
-        className.clear().append("RtlSetTypeInfo");
         arguments.append(",&");
-        childType = buildRtlType(arguments, type->queryChildType(), 0);
+        childType = buildRtlType(arguments, type->queryChildType());
         break;
     case type_unicode:
-        className.clear().append("RtlUnicodeTypeInfo");
-        arguments.append(", \"").append(type->queryLocale()).append("\"").toLowerCase();
-        length = type->getStringLen();
-        break;
     case type_varunicode:
-        className.clear().append("RtlVarUnicodeTypeInfo");
-        arguments.append(", \"").append(type->queryLocale()).append("\"").toLowerCase();
-        length = type->getStringLen();
-        break;
     case type_utf8:
-        className.clear().append("RtlUtf8TypeInfo");
-        arguments.append(", \"").append(type->queryLocale()).append("\"").toLowerCase();
-        length = type->getStringLen();
-        break;
-    case type_blob:
-    case type_pointer:
-    case type_class:
-    case type_array:
-    case type_void:
-    case type_alien:
-    case type_none:
-    case type_any:
-    case type_pattern:
-    case type_rule:
-    case type_token:
-    case type_feature:
-    case type_event:
-    case type_null:
-    case type_scope:
-    case type_transform:
-    default:
-        className.append("RtlUnimplementedTypeInfo");
-        fieldType |= (RFTMcontainsunknown|RFTMinvalidxml);
+        arguments.append(", \"").append(info.locale).append("\"").toLowerCase();
         break;
     }
-    fieldType |= (childType & (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr));
+    info.fieldType |= (childType & RFTMinherited);
 
     StringBuffer definition;
-    definition.append("const ").append(className).append(" ").append(name).append("(0x").appendf("%x", fieldType).append(",").append(length).append(arguments).append(");");
+    definition.append("const ").append(info.className).append(" ").append(name).append("(0x").appendf("%x", info.fieldType).append(",").append(info.length).append(arguments).append(");");
 
     BuildCtx typectx(declarectx);
     typectx.setNextPriority(TypeInfoPrio);
     typectx.addQuoted(definition);
 
     OwnedHqlExpr nameExpr = createVariable(name.str(), makeVoidType());
-    OwnedHqlExpr mapped = createAttribute(fieldAtom, LINK(nameExpr), getSizetConstant(fieldType));
+    OwnedHqlExpr mapped = createAttribute(fieldAtom, LINK(nameExpr), getSizetConstant(info.fieldType));
     declarectx.associateExpr(search, mapped);
     instanceName.append(name);
-    return fieldType;
+    return info.fieldType;
 }
 
 
@@ -4250,7 +4107,7 @@ void HqlCppTranslator::buildMetaInfo(MetaInstance & instance)
             assertex(!instance.isGrouped());
 
             StringBuffer typeName;
-            unsigned recordTypeFlags = buildRtlType(typeName, record->queryType(), 0);
+            unsigned recordTypeFlags = buildRtlType(typeName, record->queryType());
             s.clear().append("virtual const RtlTypeInfo * queryTypeInfo() const { return &").append(typeName).append("; }");
             metactx.addQuoted(s);
 

ecl/hqlcpp/hqltcppc.cpp

@@ -3162,7 +3162,7 @@ bool ColumnToOffsetMap::buildReadAhead(HqlCppTranslator & translator, BuildCtx &
 void ColumnToOffsetMap::buildAccessor(StringBuffer & accessorName, HqlCppTranslator & translator, BuildCtx & declarectx, IHqlExpression * selector)
 {
     StringBuffer typeName;
-    translator.buildRtlType(typeName, record->queryType(), 0);
+    translator.buildRtlType(typeName, record->queryType());
 
     BuildCtx ctx(declarectx);
     ctx.setNextPriority(TypeInfoPrio);

rtl/eclrtl/rtldynfield.cpp

@@ -23,3 +23,480 @@
 #include "eclhelper.hpp"
 #include "eclrtl_imp.hpp"
 #include "rtldynfield.hpp"
+
+//---------------------------------------------------------------------------------------------------------------------
+
+const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo() const
+{
+    const RtlTypeInfo *ret = nullptr;
+    switch (fieldType & RFTMkind)
+    {
+    case type_boolean:
+        ret = new RtlBoolTypeInfo(fieldType, length);
+        break;
+    case type_int:
+        ret = new RtlIntTypeInfo(fieldType, length);
+        break;
+    case type_real:
+        ret = new RtlRealTypeInfo(fieldType, length);
+        break;
+    case type_decimal:
+        ret = new RtlDecimalTypeInfo(fieldType, length);
+        break;
+    case type_string:
+        ret = new RtlStringTypeInfo(fieldType, length);
+        break;
+    case type_bitfield:
+        ret = new RtlBitfieldTypeInfo(fieldType, length);
+        break;
+    case type_varstring:
+        ret = new RtlVarStringTypeInfo(fieldType, length);
+        break;
+    case type_data:
+        ret = new RtlDataTypeInfo(fieldType, length);
+        break;
+    case type_table:
+        assert(childType);
+        ret = new RtlDatasetTypeInfo(fieldType, length, childType);
+        break;
+    case type_set:
+        assert(childType);
+        ret = new RtlSetTypeInfo(fieldType, length, childType);
+        break;
+    case type_row:
+        assert(childType);
+        ret = new RtlRowTypeInfo(fieldType, length, childType);
+        break;
+    case type_swapint:
+        ret = new RtlSwapIntTypeInfo(fieldType, length);
+        break;
+    case type_packedint:
+        ret = new RtlPackedIntTypeInfo(fieldType, length);
+        break;
+    case type_qstring:
+        ret = new RtlQStringTypeInfo(fieldType, length);
+        break;
+    case type_unicode:
+        ret = new RtlUnicodeTypeInfo(fieldType, length, locale);
+        break;
+    case type_varunicode:
+        ret = new RtlVarUnicodeTypeInfo(fieldType, length, locale);
+        break;
+    case type_utf8:
+        ret = new RtlUtf8TypeInfo(fieldType, length, locale);
+        break;
+    case type_record:
+        ret = new RtlRecordTypeInfo(fieldType, length, fieldsArray);
+        break;
+    default:
+        throwUnexpected();
+    }
+    return ret;
+};
+
+typedef MapBetween<const RtlTypeInfo *, const RtlTypeInfo *, StringAttr, const char *> TypeNameMap;
+
+/**
+ * class CRtlFieldTypeSerializer
+ *
+ * Serializer class for creating json representation of a RtlTypeInfo structure.
+ *
+ */
+
+class CRtlFieldTypeSerializer
+{
+public:
+    /**
+     * Serialize a RtlTypeInfo structure to JSON
+     *
+     * @param  out  Buffer for resulting serialized string
+     * @param  type RtlTypeInfo structure to be serialized
+     * @return Referenced to supplied buffer
+     */
+     static StringBuffer &serialize(StringBuffer &out, const RtlTypeInfo *type)
+     {
+         CRtlFieldTypeSerializer s(out, type);
+         s.doSerialize();
+         return out;
+     }
+private:
+    CRtlFieldTypeSerializer(StringBuffer &_out, const RtlTypeInfo *_base)
+    : json(_out), base(_base)
+    {
+    }
+    void doSerialize()
+    {
+        json.append("{");
+        serializeType(base);
+        json.append("\n}");
+    }
+    void serializeType(const RtlTypeInfo *type)
+    {
+        if (!serialized(type))
+        {
+            // Make sure all child types are serialized first
+            const RtlTypeInfo *child = type->queryChildType();
+            if (child)
+                serializeType(child);
+            const RtlFieldInfo * const * fields = type->queryFields();
+            if (fields)
+            {
+                for (;;)
+                {
+                    const RtlFieldInfo * child = *fields;
+                    if (!child)
+                        break;
+                    serializeType(child->type);
+                    fields++;
+                }
+            }
+            // Now serialize this one
+            if (type != base)
+            {
+                VStringBuffer newName("ty%d", ++nextTypeName);
+                types.setValue(type, newName.str());
+                startField(newName.str());
+                serializeMe(type);
+                closeCurly();
+            }
+            else
+                serializeMe(type);
+        }
+    }
+
+    void serializeMe(const RtlTypeInfo *type)
+    {
+        if (!type->canSerialize())
+            throw makeStringException(MSGAUD_user, 1000, "IFBLOCK and DICTIONARY type structures cannot be serialized");
+        addPropHex("fieldType", type->fieldType);
+        addProp("length", type->length);
+        addPropNonEmpty("locale", type->queryLocale());
+        const RtlTypeInfo *child = type->queryChildType();
+        if (child)
+            addPropType("child", child);
+        const RtlFieldInfo * const * fields = type->queryFields();
+        if (fields)
+        {
+            startFields();
+            for (;;)
+            {
+                const RtlFieldInfo * child = *fields;
+                if (!child)
+                    break;
+                newline();
+                openCurly();
+                addProp("name", child->name);
+                addPropType("type", child->type);
+                addProp("xpath", child->xpath);
+                if (child->flags)
+                    addPropHex("flags", child->flags);
+                // initializer is tricky - it's not (in general) a null-terminated string but the actual length is not easily available
+                if (child->initializer)
+                {
+                    addProp("init", child->type->size(child->initializer, nullptr), child->initializer);
+                }
+                closeCurly();
+                fields++;
+            }
+            endFields();
+        }
+    }
+    bool serialized(const RtlTypeInfo *type)
+    {
+        return types.find(type) != nullptr;
+    }
+    void startField(const char *name)
+    {
+        newline().appendf("\"%s\": ", name);
+        openCurly();
+    }
+    void addProp(const char *propName, const char *propVal)
+    {
+        if (propVal)
+        {
+            newline();
+            encodeJSON(json.append("\""), propName).append("\": ");
+            encodeJSON(json.append("\""), propVal).append("\"");
+        }
+    }
+    void addProp(const char *propName, size32_t propLen, const byte *propVal)
+    {
+        if (propVal)
+        {
+            newline();
+            encodeJSON(json.append("\""), propName).append("\": \"");
+            JBASE64_Encode(propVal, propLen, json, false);
+            json.append("\"");
+        }
+    }
+    void addPropNonEmpty(const char *propName, const char *propVal)
+    {
+        if (propVal && *propVal)
+            addProp(propName, propVal);
+    }
+    void addProp(const char *propName, unsigned propVal)
+    {
+        newline().appendf("\"%s\": %u", propName, propVal);
+    }
+    void addPropHex(const char *propName, unsigned propVal)
+    {
+        newline().appendf("\"%s\": %u", propName, propVal);  // Nice idea but json does not support hex constants :(
+    }
+    void addPropType(const char *propName, const RtlTypeInfo *type)
+    {
+        addProp(propName, queryTypeName(type));
+    }
+    const char *queryTypeName(const RtlTypeInfo *type)
+    {
+        StringAttr *typeName = types.getValue(type);
+        assertex(typeName);
+        return typeName->get();
+    }
+    void startFields()
+    {
+        newline().appendf("\"fields\": ");
+        openCurly('[');
+    }
+    void endFields()
+    {
+        closeCurly(']');
+    }
+    StringBuffer &newline()
+    {
+        if (commaPending)
+            json.append(',');
+        json.appendf("\n%*s", indent, "");
+        commaPending = true;
+        return json;
+    }
+    void closeCurly(char brace = '}')
+    {
+        indent--;
+        json.appendf("\n%*s%c", indent, "", brace);
+        commaPending = true;
+    }
+    void openCurly(char brace = '{')
+    {
+        json.append(brace);
+        indent++;
+        commaPending = false;
+    }
+
+    TypeNameMap types;
+    StringBuffer &json;
+    const RtlTypeInfo *base = nullptr;
+    unsigned indent = 1;
+    unsigned nextTypeName = 0;
+    bool commaPending = false;
+};
+
+/**
+ * class CRtlFieldTypeDeserializer
+ *
+ * Deserializer class for creating a RtlTypeInfo structure from json representation.
+ *
+ * Note that the resulting RtlTypeInfo structures are owned by this object and will be
+ * destroyed when this object is destroyed.
+ *
+ */
+
+class CRtlFieldTypeDeserializer : public CInterfaceOf<IRtlFieldTypeDeserializer>
+{
+public:
+    /**
+     * CRtlFieldTypeDeserializer constructor
+     *
+     */
+    CRtlFieldTypeDeserializer()
+    {
+    }
+    /**
+     * CRtlFieldTypeDeserializer destructor
+     * <p>
+     * Releases all RtlTypeInfo and related structures created by this deserializer
+     */
+    ~CRtlFieldTypeDeserializer()
+    {
+        // Need some care - all the RtlTypeInfo objects I created need to be destroyed, together with anything else I had to create
+        // Strings (other than the init strings) are preserved in the AtomTable
+        HashIterator allTypes(types);
+        ForEach(allTypes)
+        {
+            const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
+            cleanupType(*type);
+        }
+        cleanupType(base);
+    }
+    /**
+     * Obtain the deserialized type information
+     * <p>
+     * Note that the RtlTypeInfo objects are not link-counted, so the lifetime of these objects
+     * is determined by the lifetime of the deserializer. They will be released once the deserializer
+     * that created them is deleted.
+     * <p>
+     * Do not call more than once.
+     *
+     * @param  _json JSON text to be deserialized, as created by CRtlFieldTypeSerializer
+     * @return Deserialized type object
+     */
+    virtual const RtlTypeInfo *deserialize(const char *json) override
+    {
+        assertex(!base);
+        Owned<IPropertyTree> jsonTree = createPTreeFromJSONString(json);
+        base = deserializeType(jsonTree, jsonTree);
+        return base;
+    }
+
+    virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const ITypeInfo *type) override
+    {
+        VStringBuffer name("%p", type);
+        const RtlTypeInfo ** found = types.getValue(name);
+        if (found)
+            return *found;
+        info.locale = keep(info.locale);
+        const RtlTypeInfo * ret = info.createRtlTypeInfo(); // MORE - need to add to types to ensure freed - also would be nice to dedup?
+        types.setValue(name, ret);
+        return ret;
+    }
+
+    virtual const RtlTypeInfo *lookupType(const ITypeInfo *type) const override
+    {
+        VStringBuffer name("%p", type);
+        const RtlTypeInfo ** found = types.getValue(name);
+        if (found)
+            return *found;
+        return nullptr;
+    }
+
+    virtual const RtlFieldInfo *addFieldInfo(const char *fieldName, const char *xpath, const RtlTypeInfo *type, unsigned flags, const char *init) override
+    {
+        // MORE - we could hang onto this for cleanup, rather than assuming that we keep it via a later addType() call?
+        return new RtlFieldStrInfo(keep(fieldName), keep(xpath), type, flags, init);
+    }
+
+private:
+    KeptAtomTable atoms;     // Used to ensure proper lifetime of strings used in type structures
+    MapStringTo<const RtlTypeInfo *> types;  // Ensures structures only generated once
+    const RtlTypeInfo *base = nullptr;       // Holds the resulting type
+
+    void cleanupType(const RtlTypeInfo *type)
+    {
+        if (type)
+        {
+            // Releases all memory for a single RtlTypeInfo object
+            const RtlFieldInfo * const * fields = type->queryFields();
+            if (fields)
+            {
+                const RtlFieldInfo * const * cur = fields;
+                for (;;)
+                {
+                    const RtlFieldInfo * child = *cur;
+                    if (!child)
+                        break;
+                    // We don't need to delete other strings - they are owned by atom table.
+                    // But the initializer is decoded and thus owned by me
+                    delete child->initializer;
+                    delete child;
+                    cur++;
+                }
+                delete [] fields;
+            }
+            delete type;
+        }
+    }
+    const RtlTypeInfo *lookupType(const char *name, IPropertyTree *all)
+    {
+        const RtlTypeInfo ** found = types.getValue(name);
+        if (found)
+            return *found;
+        const RtlTypeInfo *type = deserializeType(all->queryPropTree(name), all);
+        types.setValue(name, type);
+        return type;
+    }
+    const char *keep(const char *string)
+    {
+        if (string)
+            return str(atoms.addAtom(string));
+        else
+            return nullptr;
+    }
+    const RtlTypeInfo *deserializeType(IPropertyTree *type, IPropertyTree *all)
+    {
+        FieldTypeInfoStruct info;
+        info.fieldType = type->getPropInt("fieldType");
+        info.length = type->getPropInt("length");
+        info.locale = keep(type->queryProp("locale"));
+        const char *child = type->queryProp("child");
+        if (child)
+            info.childType = lookupType(child, all);
+        if ((info.fieldType & RFTMkind) == type_record)
+        {
+            unsigned numFields = type->getCount("fields");
+            info.fieldsArray = new const RtlFieldInfo * [numFields+1];
+            info.fieldsArray[numFields] = nullptr;
+            Owned<IPropertyTreeIterator> fields = type->getElements("fields");
+            unsigned n = 0;
+            ForEach(*fields)
+            {
+                IPropertyTree &field = fields->query();
+                const char *fieldTypeName = field.queryProp("type");
+                const char *fieldName = keep(field.queryProp("name"));
+                const char *fieldXpath = keep(field.queryProp("xpath"));
+                unsigned flags = field.getPropInt("flags");
+                const char *fieldInit = field.queryProp("init");
+                if (fieldInit)
+                {
+                    StringBuffer decoded;
+                    JBASE64_Decode(fieldInit, decoded);
+                    fieldInit = decoded.detach(); // NOTE - this gets freed in cleanupType()
+                }
+                info.fieldsArray[n] = new RtlFieldStrInfo(fieldName, fieldXpath, lookupType(fieldTypeName, all), flags, fieldInit);
+                n++;
+            }
+        }
+        return info.createRtlTypeInfo();
+    }
+};
+
+extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer()
+{
+    return new CRtlFieldTypeDeserializer;
+}
+
+
+extern ECLRTL_API void dumpDatasetType(size32_t & __lenResult,char * & __result,IOutputMetaData &  metaVal,IRowStream * val)
+{
+    StringBuffer ret;
+    CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
+
+#ifdef _DEBUG
+    CRtlFieldTypeDeserializer deserializer;
+    StringBuffer ret2;
+    CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
+    assert(streq(ret, ret2));
+#endif
+
+    __lenResult = ret.length();
+    __result = ret.detach();
+}
+
+extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t)
+{
+    return CRtlFieldTypeSerializer::serialize(ret, t);
+}
+
+extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &  metaVal,const byte * val)
+{
+    StringBuffer ret;
+    CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
+
+#ifdef _DEBUG
+    CRtlFieldTypeDeserializer deserializer;
+    StringBuffer ret2;
+    CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
+    assert(streq(ret, ret2));
+#endif
+
+    __lenResult = ret.length();
+    __result = ret.detach();
+}

rtl/eclrtl/rtldynfield.hpp

@@ -22,27 +22,84 @@
 
 //These classes support the dynamic creation of type and field information
 
-struct ECLRTL_API RtlDynFieldInfo : public RtlFieldInfo
+//-------------------------------------------------------------------------------------------------------------------
+
+/*
+ * Used to represent the info that is needed to dynamically create an RtlTypeInfo object
+ */
+struct ECLRTL_API FieldTypeInfoStruct
 {
 public:
-    RtlDynFieldInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type)
-    : RtlFieldInfo(_name, _xpath, _type, nullptr)
-    {
-    }
-    ~RtlDynFieldInfo()
-    {
-        free(const_cast<char *>(name));
-        free(const_cast<char *>(xpath));
-    }
-};
+    unsigned fieldType = 0;
+    unsigned length = 0;
+    const char *locale = nullptr;
+    const char *className = nullptr;
+    const RtlTypeInfo *childType = nullptr;
+    const RtlFieldInfo * * fieldsArray = nullptr;
 
+    const RtlTypeInfo *createRtlTypeInfo() const;
+};
 
-//-------------------------------------------------------------------------------------------------------------------
+interface ITypeInfo;
 
-struct ECLRTL_API RtlDynRecordTypeInfo : public RtlRecordTypeInfo
+/**
+ *   IRtlFieldTypeDeserializer is used to manage the creation of RtlTypeInfo structures dynamically.
+ *   All created structures are owned by the deserializer and will be destroyed when the deserializer is destroyed.
+ */
+interface IRtlFieldTypeDeserializer : public IInterface
 {
-    inline RtlDynRecordTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlRecordTypeInfo(_fieldType, _length, _fields) {}
-    ~RtlDynRecordTypeInfo() { delete[] fields; }
+    /*
+     * Create RtlTypeInfo structures from a serialized json representation
+     *
+     * @param json The json representation
+     * @return     Deserialized RtlTypeInfo structure
+     */
+    virtual const RtlTypeInfo *deserialize(const char *json) = 0;
+
+    /*
+     * Create a single RtlTypeInfo structure from a FieldTypeInfoStruct
+     *
+     * @param info The information used to create the type
+     * @param key  A unique pointer used to dedup typeinfo structures
+     * @return     RtlTypeInfo structure
+     */
+    virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const ITypeInfo *key) = 0;
+    /*
+     * Check if a type has already been created for a given key
+     *
+     * @param key  A unique pointer used to dedup typeinfo structures
+     * @return     RtlTypeInfo structure, or nullptr if not yet created
+     */
+    virtual const RtlTypeInfo *lookupType(const ITypeInfo *key) const = 0;
+    /*
+     * Create RtlFieldInfo structure as part of a RtlTypeInfo tree
+     *
+     * @param fieldName Field name
+     * @param xpath     XPath
+     * @param type      Field type
+     * @param flags     Field flags
+     * @param init      Field initializer, or nullptr
+     * @return          RtlFieldInfo structure. All strings will be owned by the deserializer object
+     */
+    virtual const RtlFieldInfo *addFieldInfo(const char *fieldName, const char *xpath, const RtlTypeInfo *type, unsigned flags, const char *init) = 0;
+
 };
 
+extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer();
+
+extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t);
+
+/**
+ * Serialize metadata of supplied stream to JSON, and return it to ECL caller as a string. Used for testing serializer.
+ *
+ */
+extern ECLRTL_API void dumpDatasetType(size32_t & __lenResult,char * & __result,IOutputMetaData &  metaVal,IRowStream * val);
+
+/**
+ * Serialize metadata of supplied record to JSON, and return it to ECL caller as a string. Used for testing serializer.
+ *
+ */
+extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &  metaVal,const byte * val);
+
+
 #endif

rtl/eclrtl/rtlfield.cpp

@@ -41,7 +41,7 @@ static const char * queryXPath(const RtlFieldInfo * field)
 
 static const char * queryScalarXPath(const RtlFieldInfo * field)
 {
-    if (field->type->hasNonScalarXpath())
+    if (field->hasNonScalarXpath())
         return field->name;
     return queryXPath(field);
 }
@@ -1887,17 +1887,25 @@ __int64 RtlUnimplementedTypeInfo::getInt(const void * ptr) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
-RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char *_initializer)
-: RtlFieldInfo(_name, _xpath, _type, _initializer)
+RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags, const char *_initializer)
+: RtlFieldInfo(_name, _xpath, _type, _flags, _initializer)
 {
 }
 
+RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags)
+: RtlFieldInfo(_name, _xpath, _type, _flags, NULL)
+{
+}
 
 RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type)
-: RtlFieldInfo(_name, _xpath, _type, NULL)
+: RtlFieldInfo(_name, _xpath, _type, 0, NULL)
 {
 }
 
+RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char *_initializer)
+: RtlFieldInfo(_name, _xpath, _type, 0, _initializer)
+{
+}
 
 
 /* 

rtl/eclrtl/rtlfield.hpp

@@ -309,7 +309,7 @@ struct ECLRTL_API RtlDatasetTypeInfo : public RtlCompoundTypeInfo
 struct ECLRTL_API RtlDictionaryTypeInfo : public RtlCompoundTypeInfo
 {
     inline RtlDictionaryTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo * _child, IHThorHashLookupInfo *_hashInfo)
-    : RtlCompoundTypeInfo(_fieldType, _length, _child), hashInfo(_hashInfo) {}
+    : RtlCompoundTypeInfo(_fieldType|RFTMnoserialize, _length, _child), hashInfo(_hashInfo) {}
     IHThorHashLookupInfo * hashInfo;
 
     virtual size32_t getMinSize() const;
@@ -322,7 +322,7 @@ struct ECLRTL_API RtlDictionaryTypeInfo : public RtlCompoundTypeInfo
 
 struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
 {
-    inline RtlIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlTypeInfoBase(_fieldType, _length), fields(_fields) {}
+    inline RtlIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlTypeInfoBase(_fieldType|RFTMnoserialize, _length), fields(_fields) {}
     const RtlFieldInfo * const * fields;                // null terminated
 
     virtual bool getCondition(const byte * selfrow) const = 0;
@@ -384,7 +384,9 @@ public:
 struct ECLRTL_API RtlFieldStrInfo : public RtlFieldInfo
 {
     RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type);
-    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char * _initializer);
+    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags);
+    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags, const char * _initializer);
+    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char * _initializer);  // So old WU dlls can load (and then fail) rather than failing to load.
 };
 
 

rtl/eclrtl/rtlrecord.cpp

@@ -260,4 +260,3 @@ RtlDynRow::~RtlDynRow()
     delete [] variableOffsets;
 }
 
-//---------------------------------------------------------------------------------------------------------------------

rtl/eclrtl/rtlrecord.hpp

@@ -24,9 +24,10 @@
 #include <alloca.h>
 #endif
 
+#include "eclrtl_imp.hpp"
 #include "rtlfield.hpp"
 
-//These classe provides a relatively efficient way to access fields within a variable length record structure.
+//These classes provides a relatively efficient way to access fields within a variable length record structure.
 // Probably convert to an interface with various concrete implementations for varing degrees of complexity
 //
 // Complications:

rtl/include/eclhelper.hpp

@@ -44,8 +44,8 @@ typedef unsigned short UChar;
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      164
-#define MIN_ACTIVITY_INTERFACE_VERSION  164             //minimum value that is compatible with current interface - without using selectInterface
+#define ACTIVITY_INTERFACE_VERSION      165
+#define MIN_ACTIVITY_INTERFACE_VERSION  165             //minimum value that is compatible with current interface - without using selectInterface
 
 typedef unsigned char byte;
 
@@ -334,12 +334,14 @@ enum RtlFieldTypeMask
 
     RFTMalien               = 0x00000800,                   // this is the physical format of a user defined type, if unknown size we can't calculate it
     RFTMcontainsifblock     = 0x00000800,                   // contains an if block - if set on a record then it contains ifblocks, so can't work out field offsets.
-    RFTMhasnonscalarxpath   = 0x00001000,                   // field xpath contains only one node, and is therefore usable for naming scalar fields
+    RFTMhasnonscalarxpath   = 0x00001000,                   // field xpath contains multiple node, and is not therefore usable for naming scalar fields
 
     RFTMcontainsunknown     = 0x10000000,                   // contains a field of unknown type that we can't process properly
     RFTMinvalidxml          = 0x20000000,                   // cannot be called to generate xml
     RFTMhasxmlattr          = 0x40000000,                   // if specified, then xml output includes an attribute (recursive)
-    RFTMnoserialize         = 0x80000000,                   // cannot serialize this structure (contains ifblocks or other nasties)
+    RFTMnoserialize         = 0x80000000,                   // cannot serialize this typeinfo structure (contains ifblocks, dictionaries or other nasties)
+
+    RFTMinherited           = (RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr|RFTMnoserialize)    // These flags are recursively set on any parent records too
 };
 
 //MORE: Can we provide any more useful information about ifblocks  E.g., a pseudo field?  We can add later if actually useful.
@@ -377,7 +379,6 @@ struct RtlTypeInfo : public RtlITypeInfo
     inline bool isFixedSize() const { return (fieldType & RFTMunknownsize) == 0; }
     inline bool isLinkCounted() const { return (fieldType & RFTMlinkcounted) != 0; }
     inline bool isUnsigned() const { return (fieldType & RFTMunsigned) != 0; }
-    inline bool hasNonScalarXpath() const { return (fieldType & RFTMhasnonscalarxpath) != 0; }
     inline unsigned getDecimalDigits() const { return (length & 0xffff); }
     inline unsigned getDecimalPrecision() const { return (length >> 16); }
     inline unsigned getBitfieldIntSize() const { return (length & 0xff); }
@@ -395,13 +396,16 @@ public:
 //Core struct used for representing meta for a field.  Effectively used as an interface.
 struct RtlFieldInfo
 {
-    inline RtlFieldInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char *_initializer = NULL)
-    : name(_name), xpath(_xpath), type(_type), initializer((const byte *) _initializer) {}
+    inline RtlFieldInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags = 0, const char *_initializer = NULL)
+    : name(_name), xpath(_xpath), type(_type), flags(_type->fieldType | _flags), initializer((const byte *) _initializer) {}
 
     const char * name;
     const char * xpath;
     const RtlTypeInfo * type;
     const byte *initializer;
+    unsigned flags;
+
+    inline bool hasNonScalarXpath() const { return (flags & RFTMhasnonscalarxpath) != 0; }
 
     inline bool isFixedSize() const 
     { 

system/jlib/jstring.cpp

@@ -2169,10 +2169,13 @@ inline StringBuffer &encodeJSONChar(StringBuffer &s, const char *&ch, unsigned &
         case '\0':
             s.append("\\u0000");
             break;
+        case '\x7f':
+            s.append("\\u007f");
+            break;
         default:
             if (next >= ' ' && next < 128)
                 s.append(next);
-            else if (next < ' ' && next > 0)
+            else if (next < ' ')
                 s.append("\\u00").appendhex(next, true);
             else //json is always supposed to be utf8 (or other unicode formats)
             {

testing/regress/ecl/key/serializetypes.xml

@@ -0,0 +1,715 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>{
+ &quot;ty1&quot;: {
+  &quot;fieldType&quot;: 1028,
+  &quot;length&quot;: 0
+ },
+ &quot;ty2&quot;: {
+  &quot;fieldType&quot;: 4,
+  &quot;length&quot;: 20
+ },
+ &quot;ty3&quot;: {
+  &quot;fieldType&quot;: 4,
+  &quot;length&quot;: 5
+ },
+ &quot;ty4&quot;: {
+  &quot;fieldType&quot;: 1045,
+  &quot;length&quot;: 0,
+  &quot;child&quot;: &quot;ty1&quot;
+ },
+ &quot;ty5&quot;: {
+  &quot;fieldType&quot;: 257,
+  &quot;length&quot;: 8
+ },
+ &quot;ty6&quot;: {
+  &quot;fieldType&quot;: 4,
+  &quot;length&quot;: 48
+ },
+ &quot;ty7&quot;: {
+  &quot;fieldType&quot;: 282,
+  &quot;length&quot;: 8
+ },
+ &quot;ty8&quot;: {
+  &quot;fieldType&quot;: 1308,
+  &quot;length&quot;: 0
+ },
+ &quot;ty9&quot;: {
+  &quot;fieldType&quot;: 1040,
+  &quot;length&quot;: 0
+ },
+ &quot;ty10&quot;: {
+  &quot;fieldType&quot;: 1038,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty11&quot;: {
+  &quot;fieldType&quot;: 14,
+  &quot;length&quot;: 48
+ },
+ &quot;ty12&quot;: {
+  &quot;fieldType&quot;: 1054,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty13&quot;: {
+  &quot;fieldType&quot;: 30,
+  &quot;length&quot;: 8
+ },
+ &quot;ty14&quot;: {
+  &quot;fieldType&quot;: 3,
+  &quot;length&quot;: 262154
+ },
+ &quot;ty15&quot;: {
+  &quot;fieldType&quot;: 1055,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty16&quot;: {
+  &quot;fieldType&quot;: 1057,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty17&quot;: {
+  &quot;fieldType&quot;: 31,
+  &quot;length&quot;: 8
+ },
+ &quot;ty18&quot;: {
+  &quot;fieldType&quot;: 33,
+  &quot;length&quot;: 8
+ },
+ &quot;ty19&quot;: {
+  &quot;fieldType&quot;: 1065,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty20&quot;: {
+  &quot;fieldType&quot;: 1037,
+  &quot;length&quot;: 188,
+  &quot;fields&quot;: [
+   {
+    &quot;name&quot;: &quot;set1&quot;,
+    &quot;type&quot;: &quot;ty4&quot;,
+    &quot;xpath&quot;: &quot;set1\u0001Item&quot;,
+    &quot;flags&quot;: 1045,
+    &quot;init&quot;: &quot;AAoAAAABAAAAMQEAAAA0&quot;
+   },
+   {
+    &quot;name&quot;: &quot;i1&quot;,
+    &quot;type&quot;: &quot;ty5&quot;,
+    &quot;flags&quot;: 257,
+    &quot;init&quot;: &quot;NwAAAAAAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s6&quot;,
+    &quot;type&quot;: &quot;ty1&quot;,
+    &quot;flags&quot;: 1028,
+    &quot;init&quot;: &quot;BAAAAEZyZWQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s48&quot;,
+    &quot;type&quot;: &quot;ty6&quot;,
+    &quot;flags&quot;: 4,
+    &quot;init&quot;: &quot;RnJlZCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg&quot;
+   },
+   {
+    &quot;name&quot;: &quot;bu&quot;,
+    &quot;type&quot;: &quot;ty7&quot;,
+    &quot;flags&quot;: 282,
+    &quot;init&quot;: &quot;AAAAAAAAEjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;pu&quot;,
+    &quot;type&quot;: &quot;ty8&quot;,
+    &quot;flags&quot;: 1308,
+    &quot;init&quot;: &quot;kjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;d6&quot;,
+    &quot;type&quot;: &quot;ty9&quot;,
+    &quot;flags&quot;: 1040,
+    &quot;init&quot;: &quot;BgAAADAxMDIwMw==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs&quot;,
+    &quot;type&quot;: &quot;ty10&quot;,
+    &quot;flags&quot;: 1038,
+    &quot;init&quot;: &quot;MDEwMjAzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs48&quot;,
+    &quot;type&quot;: &quot;ty11&quot;,
+    &quot;flags&quot;: 14,
+    &quot;init&quot;: &quot;MDEwMjAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs&quot;,
+    &quot;type&quot;: &quot;ty12&quot;,
+    &quot;flags&quot;: 1054,
+    &quot;init&quot;: &quot;BwAAAMUEUEkEwA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs8&quot;,
+    &quot;type&quot;: &quot;ty13&quot;,
+    &quot;flags&quot;: 30,
+    &quot;init&quot;: &quot;xQRQSQTA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;dec&quot;,
+    &quot;type&quot;: &quot;ty14&quot;,
+    &quot;flags&quot;: 3,
+    &quot;init&quot;: &quot;AAASNFZ/&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u1&quot;,
+    &quot;type&quot;: &quot;ty15&quot;,
+    &quot;flags&quot;: 1055,
+    &quot;init&quot;: &quot;BgAAAKwgRQB1AHIAbwBzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu&quot;,
+    &quot;type&quot;: &quot;ty16&quot;,
+    &quot;flags&quot;: 1057,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u8&quot;,
+    &quot;type&quot;: &quot;ty17&quot;,
+    &quot;flags&quot;: 31,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAIAAgAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu8&quot;,
+    &quot;type&quot;: &quot;ty18&quot;,
+    &quot;flags&quot;: 33,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAAgACAA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;utf&quot;,
+    &quot;type&quot;: &quot;ty19&quot;,
+    &quot;flags&quot;: 1065,
+    &quot;init&quot;: &quot;BgAAAOKCrEV1cm9z&quot;
+   }
+  ]
+ },
+ &quot;ty21&quot;: {
+  &quot;fieldType&quot;: 1044,
+  &quot;length&quot;: 0,
+  &quot;child&quot;: &quot;ty20&quot;
+ },
+ &quot;ty22&quot;: {
+  &quot;fieldType&quot;: 1073742861,
+  &quot;length&quot;: 390,
+  &quot;fields&quot;: [
+   {
+    &quot;name&quot;: &quot;s5&quot;,
+    &quot;type&quot;: &quot;ty3&quot;,
+    &quot;xpath&quot;: &quot;@s5&quot;,
+    &quot;flags&quot;: 1073741828
+   },
+   {
+    &quot;name&quot;: &quot;s5a&quot;,
+    &quot;type&quot;: &quot;ty3&quot;,
+    &quot;flags&quot;: 4
+   },
+   {
+    &quot;name&quot;: &quot;grandchild&quot;,
+    &quot;type&quot;: &quot;ty21&quot;,
+    &quot;xpath&quot;: &quot;grandchild\u0001Row&quot;,
+    &quot;flags&quot;: 1044
+   },
+   {
+    &quot;name&quot;: &quot;rrf&quot;,
+    &quot;type&quot;: &quot;ty20&quot;,
+    &quot;flags&quot;: 1037
+   },
+   {
+    &quot;name&quot;: &quot;set1&quot;,
+    &quot;type&quot;: &quot;ty4&quot;,
+    &quot;xpath&quot;: &quot;set1\u0001Item&quot;,
+    &quot;flags&quot;: 1045,
+    &quot;init&quot;: &quot;AAoAAAABAAAAMQEAAAA0&quot;
+   },
+   {
+    &quot;name&quot;: &quot;i1&quot;,
+    &quot;type&quot;: &quot;ty5&quot;,
+    &quot;flags&quot;: 257,
+    &quot;init&quot;: &quot;NwAAAAAAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s6&quot;,
+    &quot;type&quot;: &quot;ty1&quot;,
+    &quot;flags&quot;: 1028,
+    &quot;init&quot;: &quot;BAAAAEZyZWQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s48&quot;,
+    &quot;type&quot;: &quot;ty6&quot;,
+    &quot;flags&quot;: 4,
+    &quot;init&quot;: &quot;RnJlZCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg&quot;
+   },
+   {
+    &quot;name&quot;: &quot;bu&quot;,
+    &quot;type&quot;: &quot;ty7&quot;,
+    &quot;flags&quot;: 282,
+    &quot;init&quot;: &quot;AAAAAAAAEjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;pu&quot;,
+    &quot;type&quot;: &quot;ty8&quot;,
+    &quot;flags&quot;: 1308,
+    &quot;init&quot;: &quot;kjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;d6&quot;,
+    &quot;type&quot;: &quot;ty9&quot;,
+    &quot;flags&quot;: 1040,
+    &quot;init&quot;: &quot;BgAAADAxMDIwMw==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs&quot;,
+    &quot;type&quot;: &quot;ty10&quot;,
+    &quot;flags&quot;: 1038,
+    &quot;init&quot;: &quot;MDEwMjAzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs48&quot;,
+    &quot;type&quot;: &quot;ty11&quot;,
+    &quot;flags&quot;: 14,
+    &quot;init&quot;: &quot;MDEwMjAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs&quot;,
+    &quot;type&quot;: &quot;ty12&quot;,
+    &quot;flags&quot;: 1054,
+    &quot;init&quot;: &quot;BwAAAMUEUEkEwA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs8&quot;,
+    &quot;type&quot;: &quot;ty13&quot;,
+    &quot;flags&quot;: 30,
+    &quot;init&quot;: &quot;xQRQSQTA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;dec&quot;,
+    &quot;type&quot;: &quot;ty14&quot;,
+    &quot;flags&quot;: 3,
+    &quot;init&quot;: &quot;AAASNFZ/&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u1&quot;,
+    &quot;type&quot;: &quot;ty15&quot;,
+    &quot;flags&quot;: 1055,
+    &quot;init&quot;: &quot;BgAAAKwgRQB1AHIAbwBzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu&quot;,
+    &quot;type&quot;: &quot;ty16&quot;,
+    &quot;flags&quot;: 1057,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u8&quot;,
+    &quot;type&quot;: &quot;ty17&quot;,
+    &quot;flags&quot;: 31,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAIAAgAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu8&quot;,
+    &quot;type&quot;: &quot;ty18&quot;,
+    &quot;flags&quot;: 33,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAAgACAA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;utf&quot;,
+    &quot;type&quot;: &quot;ty19&quot;,
+    &quot;flags&quot;: 1065,
+    &quot;init&quot;: &quot;BgAAAOKCrEV1cm9z&quot;
+   }
+  ]
+ },
+ &quot;ty23&quot;: {
+  &quot;fieldType&quot;: 1073742868,
+  &quot;length&quot;: 0,
+  &quot;child&quot;: &quot;ty22&quot;
+ },
+ &quot;fieldType&quot;: 1073742861,
+ &quot;length&quot;: 48,
+ &quot;fields&quot;: [
+  {
+   &quot;name&quot;: &quot;stringfield&quot;,
+   &quot;type&quot;: &quot;ty1&quot;,
+   &quot;flags&quot;: 1028
+  },
+  {
+   &quot;name&quot;: &quot;field2&quot;,
+   &quot;type&quot;: &quot;ty2&quot;,
+   &quot;flags&quot;: 4
+  },
+  {
+   &quot;name&quot;: &quot;field3&quot;,
+   &quot;type&quot;: &quot;ty2&quot;,
+   &quot;flags&quot;: 4
+  },
+  {
+   &quot;name&quot;: &quot;child&quot;,
+   &quot;type&quot;: &quot;ty23&quot;,
+   &quot;xpath&quot;: &quot;child\u0001Row&quot;,
+   &quot;flags&quot;: 1073742868
+  }
+ ]
+}</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>{
+ &quot;ty1&quot;: {
+  &quot;fieldType&quot;: 1028,
+  &quot;length&quot;: 0
+ },
+ &quot;ty2&quot;: {
+  &quot;fieldType&quot;: 4,
+  &quot;length&quot;: 20
+ },
+ &quot;ty3&quot;: {
+  &quot;fieldType&quot;: 4,
+  &quot;length&quot;: 5
+ },
+ &quot;ty4&quot;: {
+  &quot;fieldType&quot;: 1045,
+  &quot;length&quot;: 0,
+  &quot;child&quot;: &quot;ty1&quot;
+ },
+ &quot;ty5&quot;: {
+  &quot;fieldType&quot;: 257,
+  &quot;length&quot;: 8
+ },
+ &quot;ty6&quot;: {
+  &quot;fieldType&quot;: 4,
+  &quot;length&quot;: 48
+ },
+ &quot;ty7&quot;: {
+  &quot;fieldType&quot;: 282,
+  &quot;length&quot;: 8
+ },
+ &quot;ty8&quot;: {
+  &quot;fieldType&quot;: 1308,
+  &quot;length&quot;: 0
+ },
+ &quot;ty9&quot;: {
+  &quot;fieldType&quot;: 1040,
+  &quot;length&quot;: 0
+ },
+ &quot;ty10&quot;: {
+  &quot;fieldType&quot;: 1038,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty11&quot;: {
+  &quot;fieldType&quot;: 14,
+  &quot;length&quot;: 48
+ },
+ &quot;ty12&quot;: {
+  &quot;fieldType&quot;: 1054,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty13&quot;: {
+  &quot;fieldType&quot;: 30,
+  &quot;length&quot;: 8
+ },
+ &quot;ty14&quot;: {
+  &quot;fieldType&quot;: 3,
+  &quot;length&quot;: 262154
+ },
+ &quot;ty15&quot;: {
+  &quot;fieldType&quot;: 1055,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty16&quot;: {
+  &quot;fieldType&quot;: 1057,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty17&quot;: {
+  &quot;fieldType&quot;: 31,
+  &quot;length&quot;: 8
+ },
+ &quot;ty18&quot;: {
+  &quot;fieldType&quot;: 33,
+  &quot;length&quot;: 8
+ },
+ &quot;ty19&quot;: {
+  &quot;fieldType&quot;: 1065,
+  &quot;length&quot;: 4294967281
+ },
+ &quot;ty20&quot;: {
+  &quot;fieldType&quot;: 1037,
+  &quot;length&quot;: 188,
+  &quot;fields&quot;: [
+   {
+    &quot;name&quot;: &quot;set1&quot;,
+    &quot;type&quot;: &quot;ty4&quot;,
+    &quot;xpath&quot;: &quot;set1\u0001Item&quot;,
+    &quot;flags&quot;: 1045,
+    &quot;init&quot;: &quot;AAoAAAABAAAAMQEAAAA0&quot;
+   },
+   {
+    &quot;name&quot;: &quot;i1&quot;,
+    &quot;type&quot;: &quot;ty5&quot;,
+    &quot;flags&quot;: 257,
+    &quot;init&quot;: &quot;NwAAAAAAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s6&quot;,
+    &quot;type&quot;: &quot;ty1&quot;,
+    &quot;flags&quot;: 1028,
+    &quot;init&quot;: &quot;BAAAAEZyZWQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s48&quot;,
+    &quot;type&quot;: &quot;ty6&quot;,
+    &quot;flags&quot;: 4,
+    &quot;init&quot;: &quot;RnJlZCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg&quot;
+   },
+   {
+    &quot;name&quot;: &quot;bu&quot;,
+    &quot;type&quot;: &quot;ty7&quot;,
+    &quot;flags&quot;: 282,
+    &quot;init&quot;: &quot;AAAAAAAAEjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;pu&quot;,
+    &quot;type&quot;: &quot;ty8&quot;,
+    &quot;flags&quot;: 1308,
+    &quot;init&quot;: &quot;kjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;d6&quot;,
+    &quot;type&quot;: &quot;ty9&quot;,
+    &quot;flags&quot;: 1040,
+    &quot;init&quot;: &quot;BgAAADAxMDIwMw==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs&quot;,
+    &quot;type&quot;: &quot;ty10&quot;,
+    &quot;flags&quot;: 1038,
+    &quot;init&quot;: &quot;MDEwMjAzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs48&quot;,
+    &quot;type&quot;: &quot;ty11&quot;,
+    &quot;flags&quot;: 14,
+    &quot;init&quot;: &quot;MDEwMjAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs&quot;,
+    &quot;type&quot;: &quot;ty12&quot;,
+    &quot;flags&quot;: 1054,
+    &quot;init&quot;: &quot;BwAAAMUEUEkEwA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs8&quot;,
+    &quot;type&quot;: &quot;ty13&quot;,
+    &quot;flags&quot;: 30,
+    &quot;init&quot;: &quot;xQRQSQTA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;dec&quot;,
+    &quot;type&quot;: &quot;ty14&quot;,
+    &quot;flags&quot;: 3,
+    &quot;init&quot;: &quot;AAASNFZ/&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u1&quot;,
+    &quot;type&quot;: &quot;ty15&quot;,
+    &quot;flags&quot;: 1055,
+    &quot;init&quot;: &quot;BgAAAKwgRQB1AHIAbwBzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu&quot;,
+    &quot;type&quot;: &quot;ty16&quot;,
+    &quot;flags&quot;: 1057,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u8&quot;,
+    &quot;type&quot;: &quot;ty17&quot;,
+    &quot;flags&quot;: 31,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAIAAgAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu8&quot;,
+    &quot;type&quot;: &quot;ty18&quot;,
+    &quot;flags&quot;: 33,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAAgACAA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;utf&quot;,
+    &quot;type&quot;: &quot;ty19&quot;,
+    &quot;flags&quot;: 1065,
+    &quot;init&quot;: &quot;BgAAAOKCrEV1cm9z&quot;
+   }
+  ]
+ },
+ &quot;ty21&quot;: {
+  &quot;fieldType&quot;: 1044,
+  &quot;length&quot;: 0,
+  &quot;child&quot;: &quot;ty20&quot;
+ },
+ &quot;ty22&quot;: {
+  &quot;fieldType&quot;: 1073742861,
+  &quot;length&quot;: 390,
+  &quot;fields&quot;: [
+   {
+    &quot;name&quot;: &quot;s5&quot;,
+    &quot;type&quot;: &quot;ty3&quot;,
+    &quot;xpath&quot;: &quot;@s5&quot;,
+    &quot;flags&quot;: 1073741828
+   },
+   {
+    &quot;name&quot;: &quot;s5a&quot;,
+    &quot;type&quot;: &quot;ty3&quot;,
+    &quot;flags&quot;: 4
+   },
+   {
+    &quot;name&quot;: &quot;grandchild&quot;,
+    &quot;type&quot;: &quot;ty21&quot;,
+    &quot;xpath&quot;: &quot;grandchild\u0001Row&quot;,
+    &quot;flags&quot;: 1044
+   },
+   {
+    &quot;name&quot;: &quot;rrf&quot;,
+    &quot;type&quot;: &quot;ty20&quot;,
+    &quot;flags&quot;: 1037
+   },
+   {
+    &quot;name&quot;: &quot;set1&quot;,
+    &quot;type&quot;: &quot;ty4&quot;,
+    &quot;xpath&quot;: &quot;set1\u0001Item&quot;,
+    &quot;flags&quot;: 1045,
+    &quot;init&quot;: &quot;AAoAAAABAAAAMQEAAAA0&quot;
+   },
+   {
+    &quot;name&quot;: &quot;i1&quot;,
+    &quot;type&quot;: &quot;ty5&quot;,
+    &quot;flags&quot;: 257,
+    &quot;init&quot;: &quot;NwAAAAAAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s6&quot;,
+    &quot;type&quot;: &quot;ty1&quot;,
+    &quot;flags&quot;: 1028,
+    &quot;init&quot;: &quot;BAAAAEZyZWQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;s48&quot;,
+    &quot;type&quot;: &quot;ty6&quot;,
+    &quot;flags&quot;: 4,
+    &quot;init&quot;: &quot;RnJlZCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg&quot;
+   },
+   {
+    &quot;name&quot;: &quot;bu&quot;,
+    &quot;type&quot;: &quot;ty7&quot;,
+    &quot;flags&quot;: 282,
+    &quot;init&quot;: &quot;AAAAAAAAEjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;pu&quot;,
+    &quot;type&quot;: &quot;ty8&quot;,
+    &quot;flags&quot;: 1308,
+    &quot;init&quot;: &quot;kjQ=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;d6&quot;,
+    &quot;type&quot;: &quot;ty9&quot;,
+    &quot;flags&quot;: 1040,
+    &quot;init&quot;: &quot;BgAAADAxMDIwMw==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs&quot;,
+    &quot;type&quot;: &quot;ty10&quot;,
+    &quot;flags&quot;: 1038,
+    &quot;init&quot;: &quot;MDEwMjAzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vs48&quot;,
+    &quot;type&quot;: &quot;ty11&quot;,
+    &quot;flags&quot;: 14,
+    &quot;init&quot;: &quot;MDEwMjAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs&quot;,
+    &quot;type&quot;: &quot;ty12&quot;,
+    &quot;flags&quot;: 1054,
+    &quot;init&quot;: &quot;BwAAAMUEUEkEwA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;qs8&quot;,
+    &quot;type&quot;: &quot;ty13&quot;,
+    &quot;flags&quot;: 30,
+    &quot;init&quot;: &quot;xQRQSQTA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;dec&quot;,
+    &quot;type&quot;: &quot;ty14&quot;,
+    &quot;flags&quot;: 3,
+    &quot;init&quot;: &quot;AAASNFZ/&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u1&quot;,
+    &quot;type&quot;: &quot;ty15&quot;,
+    &quot;flags&quot;: 1055,
+    &quot;init&quot;: &quot;BgAAAKwgRQB1AHIAbwBzAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu&quot;,
+    &quot;type&quot;: &quot;ty16&quot;,
+    &quot;flags&quot;: 1057,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAA=&quot;
+   },
+   {
+    &quot;name&quot;: &quot;u8&quot;,
+    &quot;type&quot;: &quot;ty17&quot;,
+    &quot;flags&quot;: 31,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAIAAgAA==&quot;
+   },
+   {
+    &quot;name&quot;: &quot;vu8&quot;,
+    &quot;type&quot;: &quot;ty18&quot;,
+    &quot;flags&quot;: 33,
+    &quot;init&quot;: &quot;rCBFAHUAcgBvAHMAAAAgACAA&quot;
+   },
+   {
+    &quot;name&quot;: &quot;utf&quot;,
+    &quot;type&quot;: &quot;ty19&quot;,
+    &quot;flags&quot;: 1065,
+    &quot;init&quot;: &quot;BgAAAOKCrEV1cm9z&quot;
+   }
+  ]
+ },
+ &quot;ty23&quot;: {
+  &quot;fieldType&quot;: 1073742868,
+  &quot;length&quot;: 0,
+  &quot;child&quot;: &quot;ty22&quot;
+ },
+ &quot;fieldType&quot;: 1073742861,
+ &quot;length&quot;: 48,
+ &quot;fields&quot;: [
+  {
+   &quot;name&quot;: &quot;stringfield&quot;,
+   &quot;type&quot;: &quot;ty1&quot;,
+   &quot;flags&quot;: 1028
+  },
+  {
+   &quot;name&quot;: &quot;field2&quot;,
+   &quot;type&quot;: &quot;ty2&quot;,
+   &quot;flags&quot;: 4
+  },
+  {
+   &quot;name&quot;: &quot;field3&quot;,
+   &quot;type&quot;: &quot;ty2&quot;,
+   &quot;flags&quot;: 4
+  },
+  {
+   &quot;name&quot;: &quot;child&quot;,
+   &quot;type&quot;: &quot;ty23&quot;,
+   &quot;xpath&quot;: &quot;child\u0001Row&quot;,
+   &quot;flags&quot;: 1073742868
+  }
+ ]
+}</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>true</Result_3></Row>
+</Dataset>

testing/regress/ecl/serializetypes.ecl

@@ -0,0 +1,60 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test serialization of various record/dataset types, including default values
+
+s := service
+   string dumpRecordType(row val) : eclrtl,pure,library='eclrtl',entrypoint='dumpRecordType',passParameterMeta(true),fold;
+   string dumpRecordTypeNF(row val) : eclrtl,pure,library='eclrtl',entrypoint='dumpRecordType',passParameterMeta(true);
+end;
+
+rr := record
+  set of string set1 { default(['1','4']) };
+  unsigned i1 { default(55) };
+  string s6 { default('Fred') };
+  string48 s48 { default('Fred') };
+  big_endian unsigned bu { default(0x1234) };
+  packed unsigned pu { default(0x1234) };
+  data d6 { default(d'010203') };
+  varstring vs { default('010203') };
+  varstring48 vs48 { default('010203') };
+  qstring qs { default('q010203') };
+  qstring8 qs8 { default('q010203') };
+  decimal10_4 dec { default(123.4567) };
+  unicode u1 { default(u'€Euros') };
+  varunicode vu { default(u'€Euros') };
+  unicode8 u8 { default(u'€Euros') };
+  varunicode8 vu8 { default(u'€Euros') };
+  utf8 utf { default(u'€Euros') };
+end;
+
+r := record
+  string5 s5 { xpath('@s5') };
+  string5 s5a;
+  embedded dataset(rr) grandchild;
+  rr rrf;
+  rr;
+end;
+
+d := dataset([],{ string stringField; string20 field2, string20 field3, embedded dataset(r) child });
+
+f := s.dumpRecordType(d[1]);     // folded
+nf := s.dumpRecordTypeNF(d[1]);  // not folded
+
+f;
+nf;
+f = nf;