HPCC-18566 Add support for serializing ifblocks

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/thorhelper/thorfile.cpp

@@ -24,6 +24,7 @@
 #include "eclrtl_imp.hpp"
 #include "rtlfield.hpp"
 #include "rtlds_imp.hpp"
+#include "rtldynfield.hpp"
 
 #include "eclhelper_base.hpp"
 #include "thorcommon.ipp"
@@ -33,6 +34,16 @@ void setExpiryTime(IPropertyTree & properties, unsigned expireDays)
     properties.setPropInt("@expireDays", expireDays);
 }
 
+void setRtlFormat(IPropertyTree & properties, IOutputMetaData * meta)
+{
+    if (meta && meta->queryTypeInfo())
+    {
+        MemoryBuffer out;
+        if (dumpTypeInfo(out, meta->querySerializedDiskMeta()->queryTypeInfo()))
+            properties.setPropBin("_rtlType", out.length(), out.toByteArray());
+    }
+}
+
 
 class DiskWorkUnitReadArg : public CThorDiskReadArg
 {

common/thorhelper/thorfile.hpp

@@ -27,5 +27,6 @@
 
 THORHELPER_API void setExpiryTime(IPropertyTree & properties, unsigned expireDays);
 THORHELPER_API IHThorDiskReadArg * createWorkUnitReadArg(const char * filename, IHThorWorkunitReadArg * wuRead);
+THORHELPER_API void setRtlFormat(IPropertyTree & properties, IOutputMetaData * meta);
 
 #endif

ecl/hql/CMakeLists.txt

@@ -45,6 +45,7 @@ set (   SRCS
         hqlesp.cpp
         hqlexpr.cpp
         hqlfield.cpp
+        hqlfilter.cpp
         hqlfold.cpp
         hqlgram2.cpp
         hqlmanifest.cpp
@@ -82,6 +83,7 @@ set (   SRCS
         hqlesp.hpp
         hqlexpr.hpp
         hqlfield.hpp
+        hqlfilter.hpp
         hqlfold.hpp
         hqlgram.hpp
         hqlmeta.hpp

ecl/hql/hqlerrors.hpp

@@ -500,6 +500,7 @@
 #define HQLWRN_DFSlookupTypeMismatch            3147
 #define HQLWRN_NoFieldsMatch                    3148
 #define HQLWRN_DFSdenied                        3149
+#define HQLERR_NonConstantRange                 3150
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contains an illegal cycle/recursive definition %s"
@@ -543,10 +544,30 @@
 #define HQLWRN_DFSlookupFailure_Text            "Error in DFS file resolution"
 #define HQLERR_DFSlookupFailure_Text            "Failed to resolve record information in DFS for file %s"
 #define HQLERR_DFSlookupIncompatible_Text       "Resolved record information is not compatible file %s"
+#define HQLERR_NonConstantRange_Text            "Non constant substrings not supported"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */
 
+//Migrated from hqlcpp - error numbers staying the same
+
+#define HQLERR_SubstringOutOfRange              4021
+#define HQLERR_WildNotReferenceIndex            4090
+#define HQLERR_LookupNotActiveDataset           4096
+#define HQLERR_KeyedJoinTooComplex              4097
+#define HQLERR_KeyAccessNeedCast                4098
+#define HQLERR_KeyAccessNoKeyField              4099
+#define HQLERR_OrMultipleKeyfields              4129
+
+#define HQLERR_SubstringOutOfRange_Text         "Substring index %d is outside the field range"
+#define HQLERR_WildNotReferenceIndex_Text       "WILD() does not reference fields in key %s"
+#define HQLERR_LookupNotActiveDataset_Text      "Attempting to lookup field %s in a dataset which has no active element"
+#define HQLERR_KeyedJoinTooComplex_Text         "Key condition (%s) is too complex, it cannot be done with the key."
+#define HQLERR_KeyAccessNeedCast_Text           "Key condition (%s) requires casts on comparison of field '%s'"
+#define HQLERR_KeyAccessNoKeyField_Text         "Key condition (%s) does not have any comparisons against key fields"
+#define HQLERR_OrMultipleKeyfields_Text         "Cannot OR together conditions on multiple key fields (%s)"
+
+
 /////////////////////////////////////////////////////////////////////////////
 /* Code Generation errors - defined in hqlcerrors.hpp */
 #define ERR_CODEGEN_FIRST           4000

ecl/hql/hqlfilter.hpp

@@ -0,0 +1,182 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+#ifndef __HQLFILTER_HPP_
+#define __HQLFILTER_HPP_
+
+enum KeyedKind { KeyedYes, KeyedNo, KeyedExtend };
+struct HQL_API KeyCondition : public CInterface
+{
+public:
+    KeyCondition()          { keyedKind = KeyedNo; isWild = false; generated = false; wasKeyed = false; }
+    KeyCondition(IHqlExpression * _selector, IHqlExpression * _expr, KeyedKind _keyedKind)
+                            { selector.set(_selector); expr.set(_expr); keyedKind = _keyedKind; isWild = false; generated = false; wasKeyed = isKeyed(); }
+
+    bool isKeyed()          { return (keyedKind != KeyedNo); }
+
+    HqlExprAttr     selector;
+    HqlExprAttr     expr;
+    KeyedKind       keyedKind;
+    bool            isWild;
+    bool            generated;
+    bool            wasKeyed;
+
+};
+
+typedef CIArrayOf<KeyCondition> KeyConditionArray;
+
+class HQL_API KeyConditionInfo : public CInterface
+{
+public:
+    void appendPreFilter(IHqlExpression * expr) { extendAndCondition(preFilter, expr); }
+    void appendPostFilter(IHqlExpression * expr) { extendAndCondition(postFilter, expr); }
+    void appendCondition(KeyCondition & next) { conditions.append(next); }
+
+    IHqlExpression * createConjunction();
+    bool isSingleMatchCondition() const;
+
+public:
+    HqlExprAttr preFilter;          // before activity executed
+    HqlExprAttr postFilter;         // after candidate record returned
+    KeyConditionArray conditions;
+};
+
+//---------------------------------------------------------------------------
+
+enum KeyFailureReason { KFRunknown, KFRnokey, KFRor, KFRtoocomplex, KFRcast };      // ordered
+class HQL_API KeyFailureInfo
+{
+public:
+    KeyFailureInfo()  { code = KFRunknown; }
+
+    void clear()                                                    { code = KFRunknown; }
+    void merge(const KeyFailureInfo & other);
+    void reportError(IErrorReceiver & errorReceiver, IHqlExpression * condition);
+    void set(KeyFailureReason _code)                                { code = _code; }
+    void set(KeyFailureReason _code, IHqlExpression * _field)       { code = _code; field.set(_field); }
+
+protected:
+    KeyFailureReason code;
+    OwnedHqlExpr field;
+};
+
+
+enum MonitorFilterKind { NoMonitorFilter, MonitorFilterSkipEmpty, MonitorFilterSkipAll };
+
+struct HQL_API KeySelectorInfo
+{
+public:
+    KeySelectorInfo(KeyedKind _keyedKind, IHqlExpression * _selector, IHqlExpression * _expandedSelector, unsigned _fieldIdx, size32_t _offset, size32_t _size)
+    {
+        keyedKind = _keyedKind;
+        selector = _selector;
+        expandedSelector = _expandedSelector;
+        fieldIdx = _fieldIdx;
+        offset = _offset;
+        size = _size;
+    }
+
+    const char * getFFOptions();
+
+    IHqlExpression * selector;
+    IHqlExpression * expandedSelector;
+    unsigned fieldIdx;
+    size32_t offset;
+    size32_t size;
+    KeyedKind keyedKind;
+};
+
+class HQL_API FilterExtractor
+{
+public:
+    FilterExtractor(IErrorReceiver & _errorReceiver, IHqlExpression * _tableExpr, int _numKeyableFields, bool isDiskRead, bool forceValueSets);
+
+    void appendFilter(IHqlExpression * expr)                { keyed.appendPostFilter(expr); }
+    void extractFilters(IHqlExpression * filter, SharedHqlExpr & extraFilter);
+    void extractFilters(HqlExprArray & exprs, SharedHqlExpr & extraFilter);
+    void extractFiltersFromFilterDs(IHqlExpression * expr);
+    void extractAllFilters(IHqlExpression * filter);
+    IHqlExpression * queryExtraFilter()                     { return keyed.postFilter; }
+    IHqlExpression * getClearExtraFilter()                  { return keyed.postFilter.getClear(); }
+    bool isCleanlyKeyedExplicitly()                         { return cleanlyKeyedExplicitly; }
+    bool isKeyedExplicitly()                                { return keyedExplicitly; }
+    bool isFiltered()                                       { return keyed.postFilter || isKeyed(); }
+    bool isKeyed();
+    IHqlExpression * queryGlobalGuard()                     { return keyed.preFilter; }
+    void reportFailureReason(IHqlExpression * cond);
+    const char * queryKeyName(StringBuffer & s);
+    void preventMerge(IHqlExpression * select)              { if (select) noMergeSelects.append(*select); }
+
+    bool isEqualityFilterBefore(IHqlExpression * select);
+    unsigned queryKeySelectIndex(IHqlExpression * select)   { return keyableSelects.find(*select); }
+    bool isSingleMatchCondition() const;
+
+    IFieldFilter * createSingleFieldFilter(IRtlFieldTypeDeserializer &deserializer) const;
+    IFieldFilter * createFieldFilter(IRtlFieldTypeDeserializer &deserializer, IHqlExpression * selector) const;
+
+protected:
+    IHqlExpression * castToFieldAndBack(IHqlExpression * left, IHqlExpression * right);
+    bool containsTableSelects(IHqlExpression * expr);
+    IHqlExpression * createRangeCompare(IHqlExpression * selector, IHqlExpression * value, IHqlExpression * lengthExpr, bool compareEqual);
+    KeyCondition * createTranslatedCondition(IHqlExpression * cond, KeyedKind keyedKind);
+    bool extractBoolFieldFilter(KeyConditionInfo & matches, IHqlExpression * selector, KeyedKind keyedKind, bool compareValue);
+    bool extractFilters(KeyConditionInfo & matches, IHqlExpression * filter, KeyedKind keyedKind);
+    void extractFoldedWildFields(IHqlExpression * expr);
+    bool extractIfFilter(KeyConditionInfo & matches, IHqlExpression * expr, KeyedKind keyedKind);
+    bool extractSimpleCompareFilter(KeyConditionInfo & state, IHqlExpression * expr, KeyedKind keyedKind);
+    void expandKeyableFields();
+    void expandSelects(IHqlExpression * expr, IHqlSimpleScope * expandedScope, IHqlExpression * keySelector, IHqlExpression * expandedSelector);;
+    bool extractOrFilter(KeyConditionInfo & matches, IHqlExpression * filter, KeyedKind keyedKind);
+    IHqlExpression * invertTransforms(IHqlExpression * left, IHqlExpression * right);
+    bool isEqualityFilter(IHqlExpression * select);
+    bool isKeySelect(IHqlExpression * select);
+    bool isIndexInvariant(IHqlExpression * expr, bool includeRoot);
+    bool isPrevSelectKeyed(IHqlExpression * select);
+    bool matchSubstringFilter(KeyConditionInfo & matches, node_operator op, IHqlExpression * left, IHqlExpression * right, KeyedKind keyedKind, bool & duplicate);
+    IHqlExpression * isKeyableFilter(IHqlExpression * left, IHqlExpression * right, bool & duplicate, node_operator compareOp, KeyFailureInfo & reason, KeyedKind keyedKind);
+    bool okToKey(IHqlExpression * select, KeyedKind keyedKind);
+    IHqlExpression * queryKeyableSelector(IHqlExpression * expr);
+    IHqlExpression * querySimpleJoinValue(IHqlExpression * field);
+    IHqlExpression * unwindConjunction(HqlExprArray & matches, IHqlExpression * expr);
+
+    virtual IHqlExpression * getRangeLimit(ITypeInfo * fieldType, IHqlExpression * lengthExpr, IHqlExpression * value, int whichBoundary);
+
+protected:
+    IErrorReceiver & errorReceiver;
+    IHqlExpression * tableExpr;
+    KeyConditionInfo keyed;
+    unsigned numKeyableFields;
+    KeyFailureInfo failReason;
+
+    HqlExprAttr keyableRecord;
+    HqlExprArray keyableSelects;
+    // expanded record + selects have bitfields/alien/varstrings expanded to a fixed size basic type.
+    HqlExprAttr expandedRecord;
+    HqlExprArray expandedSelects;
+
+    HqlExprCopyArray noMergeSelects;    // don't merge these fields (even for wildcards) because they are separate stepping fields.
+    unsigned firstOffsetField;          // first field where the keyed offset is adjusted
+    bool onlyHozedCompares;
+    bool ignoreUnkeyed;
+    bool cleanlyKeyedExplicitly;
+    bool keyedExplicitly;
+    bool allowDynamicFormatChange;
+    const bool createValueSets;
+};
+
+extern HQL_API IHqlExpression * getExplicitlyPromotedCompare(IHqlExpression * filter);
+
+#endif

ecl/hql/hqlutil.cpp

@@ -39,6 +39,7 @@
 #include "hqlerror.hpp"
 #include "hqlexpr.ipp"
 #include "hqlrepository.hpp"
+#include "hqlfilter.hpp"
 
 #define SIZET_CACHE_SIZE    5001
 #define FIXEDATTR_CACHE_SIZE 1001
@@ -9828,6 +9829,25 @@ bool checkXpathIsNonScalar(const char *xpath)
     return (strpbrk(xpath, "/?*[]<>")!=NULL); //anything other than a single tag/attr name cannot name a scalar field
 }
 
+static IFieldFilter * createIfBlockFilter(IRtlFieldTypeDeserializer &deserializer, IHqlExpression *rowRecord, IHqlExpression * ifblock)
+{
+    //See if the condition can be matched to a simple field filter
+    OwnedHqlExpr dummyDataset = createDataset(no_anon, LINK(rowRecord));
+    OwnedHqlExpr mappedCondition = replaceSelector(ifblock->queryChild(0), querySelfReference(), dummyDataset);
+    Owned <IErrorReceiver> errorReceiver = createThrowingErrorReceiver();
+
+    FilterExtractor extractor(*errorReceiver, dummyDataset, rowRecord->numChildren(), true, true);
+    OwnedHqlExpr extraFilter;
+    extractor.extractFilters(mappedCondition, extraFilter);
+
+    bool isComplex = extraFilter || !extractor.isSingleMatchCondition();
+    if (isComplex)
+        return nullptr;
+
+    return extractor.createSingleFieldFilter(deserializer);
+}
+
+
 unsigned buildRtlRecordFields(IRtlFieldTypeDeserializer &deserializer, unsigned &idx, const RtlFieldInfo * * fieldsArray, IHqlExpression *record, IHqlExpression *rowRecord)
 {
     unsigned typeFlags = 0;
@@ -9838,8 +9858,31 @@ unsigned buildRtlRecordFields(IRtlFieldTypeDeserializer &deserializer, unsigned
         switch (field->getOperator())
         {
         case no_ifblock:
-            typeFlags |= (RFTMunknownsize);
+        {
+            OwnedHqlExpr key = createValue(no_comma, LINK(rowRecord), LINK(field));
+            const RtlTypeInfo * type = deserializer.lookupType(key);
+            if (!type)
+            {
+                FieldTypeInfoStruct info;
+                info.fieldType = type_ifblock|RFTMunknownsize;
+                info.className = "RtlDynamicIfBlockTypeInfo";
+
+                IHqlExpression * record = field->queryChild(1);
+                unsigned numFields = getFlatFieldCount(record);
+                info.fieldsArray = new const RtlFieldInfo * [numFields+1];
+                unsigned idx = 0;
+                info.fieldType |= buildRtlRecordFields(deserializer, idx, info.fieldsArray, record, record);
+                info.fieldsArray[idx] = nullptr;
+
+                info.filter = createIfBlockFilter(deserializer, rowRecord, field);
+
+                type = deserializer.addType(info, key);
+            }
+            fieldsArray[idx] = deserializer.addFieldInfo(nullptr, nullptr, type, fieldFlags, nullptr);
+            typeFlags |= fieldFlags & RFTMinherited;
+            idx++;
             break;
+        }
         case no_field:
         {
             ITypeInfo *fieldType = field->queryType();

ecl/hqlcpp/CMakeLists.txt

@@ -30,6 +30,7 @@ set (    SRCS
          hqlalias.cpp
          hqlcatom.cpp
          hqlccommon.cpp 
+         hqlcfilter.cpp
          hqlckey.cpp 
          hqlcpp.cpp 
          hqlcppc.cpp 
@@ -64,6 +65,7 @@ set (    SRCS
          hqlcatom.hpp
          hqlccommon.hpp
          hqlcerrors.hpp
+         hqlcfilter.hpp
          hqlcpp.hpp
          hqlcppc.hpp
          hqlcppds.hpp

ecl/hqlcpp/hqlcerrors.hpp

@@ -44,7 +44,6 @@
 #define HQLERR_UnknownVirtualAttr               4017
 #define HQLERR_IllegalPattern                   4018
 #define HQLERR_VarSizeSortUseThor               4020
-#define HQLERR_SubstringOutOfRange              4021
 #define HQLERR_RankOnNonList                    4022
 #define HQLERR_CastInfiniteString               4023
 #define HQLERR_TooFewParameters                 4024
@@ -108,16 +107,11 @@
 #define HQLERR_CannotDetermineSizeVar           4087
 #define HQLERR_DuplicateDefinition              4088
 #define HQLERR_DuplicateDefinitionDiffType      4089
-#define HQLERR_WildNotReferenceIndex            4090
 #define HQLERR_InconsistentKeyedOpt             4091
 #define HQLERR_OptKeyedFollowsWild              4092
 #define HQLERR_KeyedCountCantNormalize          4093
 #define HQLERR_KeyedCountNotKeyed               4094
 #define HQLERR_KeyedCountNonKeyable             4095
-#define HQLERR_LookupNotActiveDataset           4096
-#define HQLERR_KeyedJoinTooComplex              4097
-#define HQLERR_KeyAccessNeedCast                4098
-#define HQLERR_KeyAccessNoKeyField              4099
 #define HQLERR_NotSupportedInsideNoThor         4102
 #define HQLERR_RegexNoTransformSupport          4103
 #define HQLERR_AccessMatchAttrInChildQuery      4104
@@ -145,7 +139,6 @@
 #define HQLERR_StepFieldNotKeyed                4126
 #define HQLERR_StepFieldNotContiguous           4127
 #define HQLERR_SortOrderMustMatchJoinFields     4128
-#define HQLERR_OrMultipleKeyfields              4129
 #define HQLERR_RowCompressRequireFixedSize      4130
 #define HQLERR_InputsAreTooComplexToUpdate      4131
 #define HQLERR_ThorDenormOnlyLeftOuterJoin      4132
@@ -355,7 +348,6 @@
 #define HQLERR_UnknownVirtualAttr_Text          "INTERNAL: Unsupported virtual attribute '%s'"
 #define HQLERR_IllegalPattern_Text              "Illegal pattern '%s..%s'"
 #define HQLERR_VarSizeSortUseThor_Text          "THOR must be used for sorting or joining datasets with variable width rows"
-#define HQLERR_SubstringOutOfRange_Text         "Substring index %d is outside the field range"
 #define HQLERR_RankOnNonList_Text               "RANK/RANKED not supported on %s"
 #define HQLERR_CastInfiniteString_Text          "Cannot cast a string of unknown length to another character set"
 #define HQLERR_TooFewParameters_Text            "Not enough parameters passed to function '%s'"
@@ -420,16 +412,11 @@
 #define HQLERR_CannotDetermineSizeVar_Text      "Cannot determine size because variable size dataset is not in scope.  Try using sizeof(x,max)"
 #define HQLERR_DuplicateDefinition_Text         "Duplicate definition of %s"
 #define HQLERR_DuplicateDefinitionDiffType_Text "Duplicate definition of %s with different type"
-#define HQLERR_WildNotReferenceIndex_Text       "WILD() does not reference fields in key %s"
 #define HQLERR_InconsistentKeyedOpt_Text        "Field %s cannot have both KEYED and KEYED,OPT conditions"
 #define HQLERR_OptKeyedFollowsWild_Text         "KEYED(%s,OPT) follows a WILD() field in key %s"
 #define HQLERR_KeyedCountCantNormalize_Text     "COUNT(,KEYED) cannot be used on a child dataset"
 #define HQLERR_KeyedCountNotKeyed_Text          "Filter for COUNT(,KEYED) did not contained KEYED() expressions"
 #define HQLERR_KeyedCountNonKeyable_Text        "KEYED COUNT used on a non-keyable dataset"
-#define HQLERR_LookupNotActiveDataset_Text      "Attempting to lookup field %s in a dataset which has no active element"
-#define HQLERR_KeyedJoinTooComplex_Text         "Key condition (%s) is too complex, it cannot be done with the key."
-#define HQLERR_KeyAccessNeedCast_Text           "Key condition (%s) requires casts on comparison of field '%s'"
-#define HQLERR_KeyAccessNoKeyField_Text         "Key condition (%s) does not have any comparisons against key fields"
 #define HQLERR_MinusOnString_Text               "unary - cannot be performed on a string"
 #define HQLERR_NotSupportedInsideNoThor_Text    "%s is not supported inside NOTHOR()"
 #define HQLERR_RegexNoTransformSupport_Text     "Regular expression parsing does not support productions - need to use tomita"
@@ -458,7 +445,6 @@
 #define HQLERR_StepFieldNotKeyed_Text           "STEPPED field %s is not keyed"
 #define HQLERR_StepFieldNotContiguous_Text      "STEPPED field %s does not follow the previous stepped field"
 #define HQLERR_SortOrderMustMatchJoinFields_Text "Merge order must match all the stepped join fields"
-#define HQLERR_OrMultipleKeyfields_Text         "Cannot OR together conditions on multiple key fields (%s)"
 #define HQLERR_RowCompressRequireFixedSize_Text "ROW compression can only be used on fixed size indexes"
 #define HQLERR_InputsAreTooComplexToUpdate_Text "UPDATE cannot be used when the inputs names are not globally constant"
 #define HQLERR_ThorDenormOnlyLeftOuterJoin_Text "THOR currently only supports LEFT OUTER denormalize"

ecl/hqlcpp/hqlcfilter.hpp

@@ -0,0 +1,104 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+#ifndef __HQLCFILTER_HPP_
+#define __HQLCFILTER_HPP_
+
+#include "hqlfilter.hpp"
+
+struct BuildFilterState
+{
+    BuildFilterState(BuildCtx & _funcctx, const char * _listName) : funcctx(_funcctx)
+    {
+        listName = _listName;
+        curFieldIdx = 0;
+        curOffset = 0;
+        wildOffset = (unsigned) -1;
+        numActiveSets = 0;
+        warnedAllConditionsWild = false;
+        doneImplicitWarning = true;
+        wildWasKeyed = false;
+    }
+
+    inline bool wildPending() { return wildOffset != (unsigned)-1; }
+    inline void clearWild() { wildOffset = (unsigned) -1; }
+
+    const char * getSetName(bool createValueSets);
+    void popSetName();
+
+//Constant while building monitors
+    BuildCtx & funcctx;
+    const char * listName;
+
+//State variables used when generating
+    OwnedHqlExpr implicitWildField;
+    unsigned numActiveSets;
+    CIArrayOf<StringAttrItem> setNames;
+    bool doneImplicitWarning;
+    bool warnedAllConditionsWild;
+    bool wildWasKeyed;
+    unsigned curFieldIdx;
+    unsigned curOffset;
+    unsigned wildOffset;
+};
+
+
+class CppFilterExtractor : public FilterExtractor
+{
+public:
+    CppFilterExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool isDiskRead, bool forceValueSets);
+
+    void buildSegments(BuildCtx & ctx, const char * listName, bool _ignoreUnkeyed);
+    bool createGroupingMonitor(BuildCtx ctx, const char * listName, IHqlExpression * select, unsigned & maxField);
+
+protected:
+    void buildEmptyKeySegment(BuildFilterState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo);
+    void buildKeySegment(BuildFilterState & buildState, BuildCtx & ctx, unsigned whichField, unsigned curSize);
+    void buildKeySegmentExpr(BuildFilterState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * target, IHqlExpression & thisKey, MonitorFilterKind filterKind);
+    void buildKeySegmentCompareExpr(BuildFilterState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * requiredSet, IHqlExpression & thisKey);
+    void buildKeySegmentInExpr(BuildFilterState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * target, IHqlExpression & thisKey, MonitorFilterKind filterKind);
+    bool buildSingleKeyMonitor(StringBuffer & createMonitorText, KeySelectorInfo & selectorInfo, BuildCtx & ctx, IHqlExpression & thisKey);
+    void callAddAll(BuildCtx & ctx, IHqlExpression * targetVar);
+    void createStringSet(BuildCtx & ctx, const char * target, unsigned size, IHqlExpression * selector);
+    KeyCondition * createTranslatedCondition(IHqlExpression * cond, KeyedKind keyedKind);
+    IHqlExpression * getMonitorValueAddress(BuildCtx & ctx, IHqlExpression * expandedSelector, IHqlExpression * value);
+    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
+    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
+
+    virtual IHqlExpression * getRangeLimit(ITypeInfo * fieldType, IHqlExpression * lengthExpr, IHqlExpression * value, int whichBoundary) override;
+
+protected:
+    void spotSegmentCSE(BuildCtx & ctx);
+
+    class SelectSpotter : public NewHqlTransformer
+    {
+    public:
+        SelectSpotter(const HqlExprArray & _selects);
+
+        void analyseExpr(IHqlExpression * expr);
+
+    public:
+        bool hasSelects;
+        const HqlExprArray & selects;
+    };
+
+protected:
+    HqlCppTranslator & translator;
+    IIdAtom * addRangeFunc;
+    IIdAtom * killRangeFunc;
+};
+
+#endif

ecl/hqlcpp/hqlckey.cpp

@@ -296,7 +296,7 @@ protected:
     HqlExprAttr     fileAccessDataset;
     HqlExprAttr     fileAccessTransform;
     HqlExprAttr     joinSeq;
-    MonitorExtractor * monitors;
+    CppFilterExtractor * monitors;
     HqlExprAttr     fileFilter;
     HqlExprAttr     leftOnlyMatch;
     HqlExprAttr     rawRhs;
@@ -1150,7 +1150,7 @@ bool KeyedJoinInfo::processFilter()
 
     //Now extract the filters from it.
     OwnedHqlExpr extra;
-    monitors = new MonitorExtractor(rawKey, translator, -(int)numPayloadFields(rawKey), false);
+    monitors = new CppFilterExtractor(rawKey, translator, -(int)numPayloadFields(rawKey), false, false);
     if (newFilter)
         monitors->extractFilters(newFilter, extra);
 

ecl/hqlcpp/hqlcpp.cpp

@@ -2219,7 +2219,32 @@ IHqlExpression * HqlCppTranslator::queryActiveActivityLocation() const
     return NULL;
 }
 
-void HqlCppTranslator::ThrowStringException(int code,const char *format, ...)
+void HqlCppTranslator::report(IError* error)
+{
+    return errorProcessor->report(error);
+}
+
+IError * HqlCppTranslator::mapError(IError * error)
+{
+    return errorProcessor->mapError(error);
+}
+
+size32_t HqlCppTranslator::errCount()
+{
+    return errorProcessor->errCount();
+}
+
+size32_t HqlCppTranslator::warnCount()
+{
+    return errorProcessor->warnCount();
+}
+
+void HqlCppTranslator::exportMappings(IWorkUnit * wu) const
+{
+    errorProcessor->exportMappings(wu);
+}
+
+void HqlCppTranslator::ThrowStringException(int code,const char *format, ...) const
 {
     IHqlExpression * location = queryActiveActivityLocation();
     if (errorProcessor && location)
@@ -12061,6 +12086,7 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &ctx, IHqlExpressi
             StringBuffer typeText;
             getFriendlyTypeStr(paramType, typeText);
             throwError1(HQLERR_EmbeddedTypeNotSupported_X, typeText.str());
+            return; // Cannot reach here, but previous throw is virtual, so the compiler cannot be sure it does not return
         }
         args.append(*createActualFromFormal(param));
         buildFunctionCall(funcctx, bindFunc, args);
@@ -12125,6 +12151,7 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &ctx, IHqlExpressi
         StringBuffer typeText;
         getFriendlyTypeStr(returnType, typeText);
         throwError1(HQLERR_EmbeddedTypeNotSupported_X, typeText.str());
+        return; // Cannot reach here, but previous throw is virtual, so the compiler cannot be sure it does not return
     }
     OwnedHqlExpr call = bindFunctionCall(returnFunc, retargs, newReturnType);
     doBuildUserFunctionReturn(funcctx, returnType, call);

ecl/hqlcpp/hqlcpp.ipp

@@ -845,7 +845,7 @@ enum PEtype {
     PETlibrary,     // a library
     PETmax };
 
-class HQLCPP_API HqlCppTranslator : implements IHqlCppTranslator, public CInterface
+class HQLCPP_API HqlCppTranslator : implements IHqlCppTranslator, public CInterface, public IErrorReceiver
 {
 //MORE: This is in serious need of refactoring....
 
@@ -855,7 +855,6 @@ class HQLCPP_API HqlCppTranslator : implements IHqlCppTranslator, public CInterf
     friend class DiskReadBuilder;
     friend class IndexReadBuilder;
     friend class FetchBuilder;
-    friend class MonitorExtractor;
     friend class NlpParseContext;
     friend class KeyedJoinInfo;
     friend class ChildGraphBuilder;
@@ -870,6 +869,14 @@ public:
     virtual bool spanMultipleCppFiles()         { return options.spanMultipleCpp; }
     virtual unsigned getNumExtraCppFiles()      { return activitiesThisCpp ? curCppFile : 0; }
 
+  //interface IErrorReceiver
+    virtual void report(IError* error) override;
+    virtual IError * mapError(IError * error) override;
+    virtual size32_t errCount() override;
+    virtual size32_t warnCount() override;
+    virtual void exportMappings(IWorkUnit * wu) const override;
+    virtual __declspec(noreturn) void ThrowStringException(int code,const char *format, ...) const override __attribute__((format(printf, 3, 4), noreturn));            // override the global function to try and add more context information
+
 //Statements.
     void buildStmt(BuildCtx & ctx, IHqlExpression * expr);
 
@@ -928,8 +935,6 @@ public:
 
 // Helper functions
 
-    __declspec(noreturn) void ThrowStringException(int code,const char *format, ...) __attribute__((format(printf, 3, 4), noreturn));            // override the global function to try and add more context information
-
     void buildAddress(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void buildBlockCopy(BuildCtx & ctx, IHqlExpression * tgt, CHqlBoundExpr & src);
     void buildClear(BuildCtx & ctx, IHqlExpression * expr);
@@ -1060,12 +1065,12 @@ public:
 
 
     IHqlExpression * getRtlFieldKey(IHqlExpression * expr, IHqlExpression * ownerRecord, bool &isPayload);
-    unsigned buildRtlField(StringBuffer & instanceName, IHqlExpression * field, IHqlExpression * rowRecord);
+    unsigned buildRtlField(StringBuffer & instanceName, IHqlExpression * field, IHqlExpression * rowRecord, const char * rowTypeName);
     unsigned buildRtlFieldType(StringBuffer & instanceName, IHqlExpression * field, IHqlExpression * rowRecord);
     unsigned buildRtlType(StringBuffer & instanceName, ITypeInfo * type);
-    unsigned buildRtlRecordFields(StringBuffer & instanceName, IHqlExpression * record, IHqlExpression * rowRecord);
-    unsigned expandRtlRecordFields(StringBuffer & fieldListText, IHqlExpression * record, IHqlExpression * rowRecord);
-    unsigned buildRtlIfBlockField(StringBuffer & instanceName, IHqlExpression * ifblock, IHqlExpression * rowRecord, bool isPayload);
+    unsigned buildRtlRecordFields(StringBuffer & instanceName, IHqlExpression * record, IHqlExpression * rowRecord, const char * rowTypeName);
+    unsigned expandRtlRecordFields(StringBuffer & fieldListText, IHqlExpression * record, IHqlExpression * rowRecord, const char * rowTypeName);
+    unsigned buildRtlIfBlockField(StringBuffer & instanceName, IHqlExpression * ifblock, IHqlExpression * rowRecord, const char * rowTypeName, bool isPayload);
 
     void buildMetaInfo(MetaInstance & instance);
     IHqlExpression * buildMetaParameter(IHqlExpression * arg);

ecl/hqlcpp/hqlcppsys.ecl

@@ -719,9 +719,9 @@ const char * cppSystemText[]  = {
     "   utf8 getProductionUtf8(unsigned4 idx) : method,entrypoint='getUtf8';",
     "   row(dummyRecord) getProductionResult(unsigned4 idx) : method,entrypoint='queryResult';",
 
-    "   addAll() : method,include='rtlkey.hpp',entrypoint='addAll';",
-    "   addRange(const data1 lo, const data1 hi) : method,include='rtlkey.hpp',entrypoint='addRange';",
-    "   killRange(const data1 lo, const data1 hi) : method,include='rtlkey.hpp',entrypoint='killRange';",
+    "   addAll() : method,entrypoint='addAll';",
+    "   addRange(const data1 lo, const data1 hi) : method,entrypoint='addRange';",
+    "   killRange(const data1 lo, const data1 hi) : method,entrypoint='killRange';",
     "   addRawRange(const data1 lo, const data1 hi) : method,entrypoint='addRawRange';",
     "   killRawRange(const data1 lo, const data1 hi) : method,entrypoint='killRawRange';",
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3643,7 +3643,7 @@ IHqlExpression * HqlCppTranslator::getRtlFieldKey(IHqlExpression * expr, IHqlExp
     return LINK(expr);
 }
 
-unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpression * field, IHqlExpression * rowRecord)
+unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpression * field, IHqlExpression * rowRecord, const char * rowTypeName)
 {
     bool isPayload = false;
     OwnedHqlExpr fieldKey = getRtlFieldKey(field, rowRecord, isPayload);
@@ -3664,7 +3664,7 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer & instanceName, IHqlExpres
     unsigned fieldFlags = 0;
     if (field->getOperator() == no_ifblock)
     {
-        typeFlags = buildRtlIfBlockField(name, field, rowRecord, isPayload);
+        typeFlags = buildRtlIfBlockField(name, field, rowRecord, rowTypeName, isPayload);
     }
     else
     {
@@ -3803,7 +3803,7 @@ unsigned HqlCppTranslator::buildRtlFieldType(StringBuffer & instanceName, IHqlEx
 }
 
 
-unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHqlExpression * ifblock, IHqlExpression * rowRecord, bool isPayload)
+unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHqlExpression * ifblock, IHqlExpression * rowRecord, const char * rowTypeName, bool isPayload)
 {
     StringBuffer typeName, s;
     BuildCtx declarectx(*code, declareAtom);
@@ -3813,30 +3813,55 @@ unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHq
     {
         unsigned length = 0;
         StringBuffer childTypeName;
-        unsigned childType = buildRtlRecordFields(childTypeName, ifblock->queryChild(1), rowRecord);
+        unsigned childType = buildRtlRecordFields(childTypeName, ifblock->queryChild(1), rowRecord, rowTypeName);
         fieldType |= (childType & RFTMinherited);
 
         StringBuffer className;
         typeName.append("ty").append(++nextTypeId);
         className.append("tyc").append(nextFieldId);
 
+        //See if the condition can be matched to a simple field filter
+        OwnedHqlExpr dummyDataset = createDataset(no_anon, LINK(rowRecord));
+        OwnedHqlExpr mappedCondition = replaceSelector(ifblock->queryChild(0), querySelfReference(), dummyDataset);
+        CppFilterExtractor extractor(dummyDataset, *this, rowRecord->numChildren(), true, true);
+        OwnedHqlExpr extraFilter;
+        extractor.extractFilters(mappedCondition, extraFilter);
+
+        bool isComplex = extraFilter || !extractor.isSingleMatchCondition();
+        const char * baseClass = isComplex ? "RtlComplexIfBlockTypeInfo" : "RtlSimpleIfBlockTypeInfo";
+        if (isComplex)
+            fieldType |= RFTMnoserialize;
+
         //The ifblock needs a unique instance of the class to evaluate the test
         BuildCtx fieldclassctx(declarectx);
+        // Ensure parent row type is forward declared.  This may possibly occur multiple times, but is harmless, and not worth addressing.
+        fieldclassctx.setNextPriority(TypeInfoPrio);
+        fieldclassctx.addQuoted(s.clear().append("extern const RtlRecordTypeInfo ").append(rowTypeName).append(";"));
+
         fieldclassctx.setNextPriority(TypeInfoPrio);
-        fieldclassctx.addQuotedCompound(s.clear().append("struct ").append(className).append(" final : public RtlIfBlockTypeInfo"), ";");
-        fieldclassctx.addQuoted(s.clear().append(className).append("() : RtlIfBlockTypeInfo(0x").appendf("%x", fieldType).append(",").append(0).append(",").append(childTypeName).append(") {}"));
+        fieldclassctx.addQuotedCompound(s.clear().appendf("struct %s final : public %s", className.str(), baseClass), ";");
+        fieldclassctx.addQuoted(s.clear().appendf("%s() : %s(0x%x,0,%s,&%s) {}", className.str(), baseClass, fieldType, childTypeName.str(), rowTypeName));
 
         OwnedHqlExpr anon = createDataset(no_anon, LINK(rowRecord));
         {
             MemberFunction deletefunc(*this, fieldclassctx, "virtual void doDelete() const override final");
             deletefunc.ctx.addQuotedLiteral("delete this;");
         }
+
+        if (isComplex)
         {
             MemberFunction condfunc(*this, fieldclassctx, "virtual bool getCondition(const byte * self) const override");
             BoundRow * self = bindTableCursor(condfunc.ctx, anon, "self");
             OwnedHqlExpr cond = self->bindToRow(ifblock->queryChild(0), querySelfReference());
             buildReturn(condfunc.ctx, cond);
         }
+        else
+        {
+            MemberFunction condfunc(*this, fieldclassctx, "virtual IFieldFilter * createCondition() const override");
+            BoundRow * self = bindTableCursor(condfunc.ctx, anon, "self");
+            OwnedHqlExpr cond = self->bindToRow(ifblock->queryChild(0), querySelfReference());
+            extractor.buildSegments(condfunc.ctx, nullptr, true);
+        }
 
         s.clear().append("const ").append(className).append(" ").append(typeName).append(";");
 
@@ -3862,7 +3887,7 @@ unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHq
 }
 
 
-unsigned HqlCppTranslator::expandRtlRecordFields(StringBuffer & fieldListText, IHqlExpression * record, IHqlExpression * rowRecord)
+unsigned HqlCppTranslator::expandRtlRecordFields(StringBuffer & fieldListText, IHqlExpression * record, IHqlExpression * rowRecord, const char * rowTypeName)
 {
     unsigned fieldType = 0;
     ForEachChild(i, record)
@@ -3873,11 +3898,11 @@ unsigned HqlCppTranslator::expandRtlRecordFields(StringBuffer & fieldListText, I
         {
         case no_field:
         case no_ifblock:
-            childType = buildRtlField(fieldListText, cur, rowRecord);
+            childType = buildRtlField(fieldListText, cur, rowRecord, rowTypeName);
             fieldListText.append(",");
             break;
         case no_record:
-            childType = expandRtlRecordFields(fieldListText, cur, rowRecord);
+            childType = expandRtlRecordFields(fieldListText, cur, rowRecord, rowTypeName);
             break;
         }
         fieldType |= (childType & RFTMinherited);
@@ -3886,10 +3911,10 @@ unsigned HqlCppTranslator::expandRtlRecordFields(StringBuffer & fieldListText, I
 }
 
 
-unsigned HqlCppTranslator::buildRtlRecordFields(StringBuffer & instanceName, IHqlExpression * record, IHqlExpression * rowRecord)
+unsigned HqlCppTranslator::buildRtlRecordFields(StringBuffer & instanceName, IHqlExpression * record, IHqlExpression * rowRecord, const char * rowTypeName)
 {
     StringBuffer fieldListText;
-    unsigned fieldFlags = expandRtlRecordFields(fieldListText, record, rowRecord);
+    unsigned fieldFlags = expandRtlRecordFields(fieldListText, record, rowRecord, rowTypeName);
 
     StringBuffer name;
     name.append("tl").append(++nextTypeId);
@@ -3943,7 +3968,7 @@ unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo *
             IHqlExpression * record = ::queryRecord(type);
             arguments.append(",");
             StringBuffer fieldsInstance;
-            childType = buildRtlRecordFields(fieldsInstance, record, record);
+            childType = buildRtlRecordFields(fieldsInstance, record, record, name);
             arguments.append(fieldsInstance);
 
             //The following code could be used to generate an extra list of fields with nested records expanded out,
@@ -10716,7 +10741,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
         bool grouped = isGrouped(dataset);
         bool ignoreGrouped = !expr->hasAttribute(groupedAtom);
         if ((kind != TAKspill) || (dataset->queryType() != expr->queryType()) || (grouped && ignoreGrouped))
-            buildMetaMember(instance->classctx, dataset, grouped && !ignoreGrouped, "queryDiskRecordSize");
+        {
+            OwnedHqlExpr serializedRecord = getSerializedForm(dataset->queryRecord(), diskAtom);
+            buildMetaMember(instance->classctx, serializedRecord, grouped && !ignoreGrouped, "queryDiskRecordSize");
+        }
         buildClusterHelper(instance->classctx, expr);
 
         //Both csv write and pipe with csv/xml format

ecl/hqlcpp/hqlsource.ipp

@@ -17,228 +17,9 @@
 #ifndef __HQLSOURCE_IPP_
 #define __HQLSOURCE_IPP_
 
-IHqlExpression * convertToPhysicalTable(IHqlExpression * tableExpr, bool ensureSerialized);
-
-enum KeyedKind { KeyedYes, KeyedNo, KeyedExtend };
-struct KeyCondition : public CInterface
-{
-public:
-    KeyCondition()          { keyedKind = KeyedNo; isWild = false; generated = false; wasKeyed = false; }
-    KeyCondition(IHqlExpression * _selector, IHqlExpression * _expr, KeyedKind _keyedKind)          
-                            { selector.set(_selector); expr.set(_expr); keyedKind = _keyedKind; isWild = false; generated = false; wasKeyed = isKeyed(); }
-
-    bool isKeyed()          { return (keyedKind != KeyedNo); }
-
-    HqlExprAttr     selector;
-    HqlExprAttr     expr;
-    KeyedKind       keyedKind;
-    bool            isWild;
-    bool            generated;
-    bool            wasKeyed;
-
-};
-
-typedef CIArrayOf<KeyCondition> KeyConditionArray;
-
-class KeyConditionInfo : public CInterface
-{
-public:
-    void appendPreFilter(IHqlExpression * expr) { extendAndCondition(preFilter, expr); }
-    void appendPostFilter(IHqlExpression * expr) { extendAndCondition(postFilter, expr); }
-    void appendCondition(KeyCondition & next) { conditions.append(next); }
-
-    IHqlExpression * createConjunction();
-
-public:
-    HqlExprAttr preFilter;          // before activity executed
-    HqlExprAttr postFilter;         // after candidate record returned
-    KeyConditionArray conditions;
-};
-
-//---------------------------------------------------------------------------
-
-enum KeyFailureReason { KFRunknown, KFRnokey, KFRor, KFRtoocomplex, KFRcast };      // ordered
-class KeyFailureInfo
-{
-public:
-    KeyFailureInfo()  { code = KFRunknown; }
-
-    void clear()                                                    { code = KFRunknown; }
-    void merge(const KeyFailureInfo & other);
-    void reportError(HqlCppTranslator & translator, IHqlExpression * condition);
-    void set(KeyFailureReason _code)                                { code = _code; }
-    void set(KeyFailureReason _code, IHqlExpression * _field)       { code = _code; field.set(_field); }
-
-protected:
-    KeyFailureReason code;
-    OwnedHqlExpr field;
-};
-    
-
-struct BuildMonitorState
-{
-    BuildMonitorState(BuildCtx & _funcctx, const char * _listName) : funcctx(_funcctx) 
-    { 
-        listName = _listName; 
-        curFieldIdx = 0;
-        curOffset = 0;
-        wildOffset = (unsigned) -1;
-        numActiveSets = 0;
-        warnedAllConditionsWild = false;
-        doneImplicitWarning = true;
-        wildWasKeyed = false;
-    }
-
-    inline bool wildPending() { return wildOffset != (unsigned)-1; }
-    inline void clearWild() { wildOffset = (unsigned) -1; }
-
-    const char * getSetName(bool createValueSets);
-    void popSetName();
-
-//Constant while building monitors
-    BuildCtx & funcctx;
-    const char * listName;
-
-//State variables used when generating
-    OwnedHqlExpr implicitWildField;
-    unsigned numActiveSets;
-    CIArrayOf<StringAttrItem> setNames;
-    bool doneImplicitWarning;
-    bool warnedAllConditionsWild;
-    bool wildWasKeyed;
-    unsigned curFieldIdx;
-    unsigned curOffset;
-    unsigned wildOffset;
-};
-
-enum MonitorFilterKind { NoMonitorFilter, MonitorFilterSkipEmpty, MonitorFilterSkipAll };
-
-struct KeySelectorInfo
-{
-public:
-    KeySelectorInfo(KeyedKind _keyedKind, IHqlExpression * _selector, IHqlExpression * _expandedSelector, unsigned _fieldIdx, size32_t _offset, size32_t _size)
-    {
-        keyedKind = _keyedKind; 
-        selector = _selector; 
-        expandedSelector = _expandedSelector;
-        fieldIdx = _fieldIdx;
-        offset = _offset; 
-        size = _size;
-    }
-
-    const char * getFFOptions();
-
-    IHqlExpression * selector;
-    IHqlExpression * expandedSelector;
-    unsigned fieldIdx;
-    size32_t offset;
-    size32_t size;
-    KeyedKind keyedKind;
-};
+#include "hqlcfilter.hpp"
 
-class MonitorExtractor
-{
-public:
-    MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool isDiskRead);
-
-    void appendFilter(IHqlExpression * expr)                { keyed.appendPostFilter(expr); }
-    void buildSegments(BuildCtx & ctx, const char * listName, bool _ignoreUnkeyed);
-    void extractFilters(IHqlExpression * filter, SharedHqlExpr & extraFilter);
-    void extractFilters(HqlExprArray & exprs, SharedHqlExpr & extraFilter);
-    void extractFiltersFromFilterDs(IHqlExpression * expr);
-    void extractAllFilters(IHqlExpression * filter);
-    IHqlExpression * queryExtraFilter()                     { return keyed.postFilter; }
-    IHqlExpression * getClearExtraFilter()                  { return keyed.postFilter.getClear(); }
-    bool isCleanlyKeyedExplicitly()                         { return cleanlyKeyedExplicitly; }
-    bool isKeyedExplicitly()                                { return keyedExplicitly; }
-    bool isFiltered()                                       { return keyed.postFilter || isKeyed(); }
-    bool isKeyed();
-    IHqlExpression * queryGlobalGuard()                     { return keyed.preFilter; }
-    void reportFailureReason(IHqlExpression * cond)         { failReason.reportError(translator, cond); }
-    const char * queryKeyName(StringBuffer & s);
-    void preventMerge(IHqlExpression * select)              { if (select) noMergeSelects.append(*select); }
-
-    bool isEqualityFilterBefore(IHqlExpression * select);
-    unsigned queryKeySelectIndex(IHqlExpression * select)   { return keyableSelects.find(*select); }
-    bool createGroupingMonitor(BuildCtx ctx, const char * listName, IHqlExpression * select, unsigned & maxField);
-
-protected:
-    void buildEmptyKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo);
-    void buildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned whichField, unsigned curSize);
-    void buildKeySegmentExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * target, IHqlExpression & thisKey, MonitorFilterKind filterKind);
-    void buildKeySegmentCompareExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * requiredSet, IHqlExpression & thisKey);
-    void buildKeySegmentInExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * target, IHqlExpression & thisKey, MonitorFilterKind filterKind);
-    bool buildSingleKeyMonitor(StringBuffer & createMonitorText, KeySelectorInfo & selectorInfo, BuildCtx & ctx, IHqlExpression & thisKey);
-    void callAddAll(BuildCtx & ctx, IHqlExpression * targetVar);
-    IHqlExpression * castToFieldAndBack(IHqlExpression * left, IHqlExpression * right);
-    bool containsTableSelects(IHqlExpression * expr);
-    IHqlExpression * createRangeCompare(IHqlExpression * selector, IHqlExpression * value, IHqlExpression * lengthExpr, bool compareEqual);
-    void createStringSet(BuildCtx & ctx, const char * target, unsigned size, IHqlExpression * selector);
-    KeyCondition * createTranslatedCondition(IHqlExpression * cond, KeyedKind keyedKind);
-    bool extractBoolFieldFilter(KeyConditionInfo & matches, IHqlExpression * selector, KeyedKind keyedKind, bool compareValue);
-    bool extractFilters(KeyConditionInfo & matches, IHqlExpression * filter, KeyedKind keyedKind);
-    void extractFoldedWildFields(IHqlExpression * expr);
-    bool extractIfFilter(KeyConditionInfo & matches, IHqlExpression * expr, KeyedKind keyedKind);
-    bool extractSimpleCompareFilter(KeyConditionInfo & state, IHqlExpression * expr, KeyedKind keyedKind);
-    void expandKeyableFields();
-    void expandSelects(IHqlExpression * expr, IHqlSimpleScope * expandedScope, IHqlExpression * keySelector, IHqlExpression * expandedSelector);;
-    bool extractOrFilter(KeyConditionInfo & matches, IHqlExpression * filter, KeyedKind keyedKind);
-    IHqlExpression * getMonitorValueAddress(BuildCtx & ctx, IHqlExpression * expandedSelector, IHqlExpression * value);
-    IHqlExpression * getRangeLimit(ITypeInfo * fieldType, IHqlExpression * lengthExpr, IHqlExpression * value, int whichBoundary);
-    IHqlExpression * invertTransforms(IHqlExpression * left, IHqlExpression * right);
-    bool isEqualityFilter(IHqlExpression * select);
-    bool isKeySelect(IHqlExpression * select);
-    bool isIndexInvariant(IHqlExpression * expr, bool includeRoot);
-    bool isPrevSelectKeyed(IHqlExpression * select);
-    bool matchSubstringFilter(KeyConditionInfo & matches, node_operator op, IHqlExpression * left, IHqlExpression * right, KeyedKind keyedKind, bool & duplicate);
-    IHqlExpression * isKeyableFilter(IHqlExpression * left, IHqlExpression * right, bool & duplicate, node_operator compareOp, KeyFailureInfo & reason, KeyedKind keyedKind);
-    bool okToKey(IHqlExpression * select, KeyedKind keyedKind);
-    IHqlExpression * queryKeyableSelector(IHqlExpression * expr);
-    IHqlExpression * querySimpleJoinValue(IHqlExpression * field);
-    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
-    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
-    IHqlExpression * unwindConjunction(HqlExprArray & matches, IHqlExpression * expr);
-
-protected:
-    void spotSegmentCSE(BuildCtx & ctx);
-
-    class SelectSpotter : public NewHqlTransformer
-    {
-    public:
-        SelectSpotter(const HqlExprArray & _selects);
-
-        void analyseExpr(IHqlExpression * expr);
-
-    public:
-        bool hasSelects;
-        const HqlExprArray & selects;
-    };
-protected:
-    IHqlExpression * tableExpr;
-    HqlCppTranslator & translator;
-//  LinkedHqlExpr filter;
-//  LinkedHqlExpr globalGuard;
-    KeyConditionInfo keyed;
-    unsigned numKeyableFields;
-    KeyFailureInfo failReason;
-
-    HqlExprAttr keyableRecord;
-    HqlExprArray keyableSelects;
-    // expanded record + selects have bitfields/alien/varstrings expanded to a fixed size basic type.
-    HqlExprAttr expandedRecord;
-    HqlExprArray expandedSelects;
-
-    HqlExprCopyArray noMergeSelects;    // don't merge these fields (even for wildcards) because they are separate stepping fields.
-    unsigned firstOffsetField;          // first field where the keyed offset is adjusted
-    bool onlyHozedCompares;
-    bool ignoreUnkeyed;
-    bool cleanlyKeyedExplicitly;
-    bool keyedExplicitly;
-    bool allowDynamicFormatChange;
-    const bool createValueSets;
-    IIdAtom * addRangeFunc;
-    IIdAtom * killRangeFunc;
-};
+IHqlExpression * convertToPhysicalTable(IHqlExpression * tableExpr, bool ensureSerialized);
 
 //---------------------------------------------------------------------------
 

ecl/hqlcpp/hqlstep.cpp

@@ -749,6 +749,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityRowsetRange(BuildCtx & ctx, IH
         }
     default:
         throwError(HQLERR_UnsupportedRowsetRangeParam);
+        return nullptr; // Cannot reach here, but previous throw is virtual, so the compiler cannot be sure it does not return
     }
 
     Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, argName);

ecl/hthor/hthor.cpp

@@ -748,12 +748,8 @@ void CHThorDiskWriteActivity::setFormat(IFileDescriptor * desc)
     const char *recordECL = helper.queryRecordECL();
     if (recordECL && *recordECL)
         desc->queryProperties().setProp("ECL", recordECL);
-    if (helper.queryDiskRecordSize()->queryTypeInfo())
-    {
-        MemoryBuffer out;
-        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
-        desc->queryProperties().setPropBin("_rtlType", out.length(), out.toByteArray());
-    }
+
+    setRtlFormat(desc->queryProperties(), helper.queryDiskRecordSize());
     desc->queryProperties().setProp("@kind", "flat");
 }
 
@@ -891,12 +887,7 @@ void CHThorCsvWriteActivity::setFormat(IFileDescriptor * desc)
     const char *recordECL = helper.queryRecordECL();
     if (recordECL && *recordECL)
         desc->queryProperties().setProp("ECL", recordECL);
-    if (helper.queryDiskRecordSize()->queryTypeInfo())
-    {
-        MemoryBuffer out;
-        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
-        desc->queryProperties().setPropBin("_rtlType", out.length(), out.toByteArray());
-    }
+    setRtlFormat(desc->queryProperties(), helper.queryDiskRecordSize());
 }
 
 //=====================================================================================================
@@ -985,12 +976,7 @@ void CHThorXmlWriteActivity::setFormat(IFileDescriptor * desc)
     const char *recordECL = helper.queryRecordECL();
     if (recordECL && *recordECL)
         desc->queryProperties().setProp("ECL", recordECL);
-    if (helper.queryDiskRecordSize()->queryTypeInfo())
-    {
-        MemoryBuffer out;
-        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
-        desc->queryProperties().setPropBin("_rtlType", out.length(), out.toByteArray());
-    }
+    setRtlFormat(desc->queryProperties(), helper.queryDiskRecordSize());
 }
 
 //=====================================================================================================
@@ -1236,12 +1222,7 @@ void CHThorIndexWriteActivity::execute()
         rtlFree(layoutMetaBuff);
     }
     // New record layout info
-    if (helper.queryDiskRecordSize()->queryTypeInfo())
-    {
-        MemoryBuffer out;
-        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
-        properties.setPropBin("_rtlType", out.length(), out.toByteArray());
-    }
+    setRtlFormat(properties, helper.queryDiskRecordSize());
 
     StringBuffer lfn;
     Owned<IDistributedFile> dfile = NULL;
@@ -1306,12 +1287,7 @@ void CHThorIndexWriteActivity::buildLayoutMetadata(Owned<IPropertyTree> & metada
     if(!metadata) metadata.setown(createPTree("metadata"));
     metadata->setProp("_record_ECL", helper.queryRecordECL());
 
-    if (helper.queryDiskRecordSize()->queryTypeInfo())
-    {
-        MemoryBuffer out;
-        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
-        metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
-    }
+    setRtlFormat(*metadata, helper.queryDiskRecordSize());
 }
 
 //=====================================================================================================

roxie/ccd/ccdserver.cpp

@@ -11689,12 +11689,7 @@ public:
         const char *recordECL = helper.queryRecordECL();
         if (recordECL && *recordECL)
             fileProps.setProp("ECL", recordECL);
-        if (helper.queryDiskRecordSize()->queryTypeInfo())
-        {
-            MemoryBuffer out;
-            if (dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo()))
-                fileProps.setPropBin("_rtlType", out.length(), out.toByteArray());
-        }
+        setRtlFormat(fileProps, helper.queryDiskRecordSize());
 
         fileProps.setProp("@kind", "flat"); // default, derivitives may override
     }
@@ -12036,12 +12031,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
             metadata.setown(createPTree("metadata", ipt_fast));
         metadata->setProp("_record_ECL", helper.queryRecordECL());
 
-        if (helper.queryDiskRecordSize()->queryTypeInfo())
-        {
-            MemoryBuffer out;
-            if (dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo()))
-                metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
-        }
+        setRtlFormat(*metadata, helper.queryDiskRecordSize());
     }
 
 public:
@@ -12250,12 +12240,7 @@ public:
             rtlFree(layoutMetaBuff);
         }
         // New record layout info
-        if (helper.queryDiskRecordSize()->queryTypeInfo())
-        {
-            MemoryBuffer out;
-            if (dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo()))
-                properties.setPropBin("_rtlType", out.length(), out.toByteArray());
-        }
+        setRtlFormat(properties, helper.queryDiskRecordSize());
     }
 
     IUserDescriptor *queryUserDescriptor() const

rtl/eclrtl/rtldynfield.cpp

@@ -117,7 +117,7 @@ const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_c
         ret = new RtlRecordTypeInfo(fieldType, length, fieldsArray);
         break;
     case type_ifblock:
-        ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray);
+        ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray, nullptr, filter);
         break;
     case type_alien:
         assert(childType);
@@ -171,9 +171,9 @@ private:
         if (!serialized(type))
         {
             // Make sure all child types are serialized first
-            const RtlTypeInfo *child = type->queryChildType();
-            if (child)
-                serializeType(child);
+            const RtlTypeInfo *childType = type->queryChildType();
+            if (childType)
+                serializeType(childType);
             const RtlFieldInfo * const * fields = type->queryFields();
             if (fields)
             {
@@ -207,9 +207,18 @@ private:
         addPropHex("fieldType", type->fieldType);
         addProp("length", type->length);
         addPropNonEmpty("locale", type->queryLocale());
-        const RtlTypeInfo *child = type->queryChildType();
-        if (child)
-            addPropType("child", child);
+        const RtlTypeInfo *childType = type->queryChildType();
+        if (childType)
+            addPropType("child", childType);
+        const IFieldFilter * filter = type->queryFilter();
+        if (filter)
+        {
+            StringBuffer filterText;
+            filter->serialize(filterText);
+            addProp("filterField", filter->queryFieldIndex());
+            addPropType("filterType", &filter->queryType());
+            addProp("filter", filterText);
+        }
         const RtlFieldInfo * const * fields = type->queryFields();
         if (fields)
         {
@@ -401,6 +410,13 @@ private:
             out.append(locale);
         if (child)
             out.appendPacked(queryTypeIdx(child));
+        const IFieldFilter * filter = type->queryFilter();
+        if (filter)
+        {
+            out.appendPacked(filter->queryFieldIndex());
+            out.appendPacked(queryTypeIdx(&filter->queryType()));
+            filter->serialize(out);
+        }
         if (fields)
         {
             unsigned count = countFields(fields);
@@ -477,6 +493,7 @@ public:
     {
         // Need some care - all the RtlTypeInfo objects I created need to be destroyed, together with anything else I had to create
         // Strings (other than the init strings) are preserved in the AtomTable
+        // First allow the types to clean up any critical cached information, then delete them in a second pass
         HashIterator allTypes(types);
         ForEach(allTypes)
         {
@@ -484,6 +501,13 @@ public:
             cleanupType(*type);
         }
         cleanupType(base);
+
+        ForEach(allTypes)
+        {
+            const RtlTypeInfo **type = types.mapToValue(&allTypes.query());
+            deleteType(*type);
+        }
+        deleteType(base);
     }
     /**
      * Obtain the deserialized type information
@@ -578,21 +602,24 @@ public:
         }
     }
 
-    virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const ITypeInfo *type) override
+    virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const IInterface *typeOrIfblock) override
     {
-        VStringBuffer name("%p", type);
+        VStringBuffer name("%p", typeOrIfblock);
         const RtlTypeInfo ** found = types.getValue(name);
         if (found)
             return *found;
         info.locale = keep(info.locale);
         const RtlTypeInfo * ret = info.createRtlTypeInfo(callback);
         types.setValue(name, ret);
+        unsigned baseType = (info.fieldType & RFTMkind);
+        if (baseType == type_record)
+            patchIfBlockParentRow(ret, static_cast<const RtlRecordTypeInfo *>(ret));
         return ret;
     }
 
-    virtual const RtlTypeInfo *lookupType(const ITypeInfo *type) const override
+    virtual const RtlTypeInfo *lookupType(const IInterface * typeOrIfBlock) const override
     {
-        VStringBuffer name("%p", type);
+        VStringBuffer name("%p", typeOrIfBlock);
         const RtlTypeInfo ** found = types.getValue(name);
         if (found)
             return *found;
@@ -610,7 +637,7 @@ private:
     MapStringTo<const RtlTypeInfo *> types;  // Ensures structures only generated once
     const RtlTypeInfo *base = nullptr;       // Holds the resulting type
     IThorIndexCallback *callback = nullptr;
-    void cleanupType(const RtlTypeInfo *type)
+    void deleteType(const RtlTypeInfo *type)
     {
         if (type)
         {
@@ -626,7 +653,7 @@ private:
                         break;
                     // We don't need to delete other strings - they are owned by atom table.
                     // But the initializer is decoded and thus owned by me
-                    delete child->initializer;
+                    free((void *)child->initializer);
                     delete child;
                     cur++;
                 }
@@ -635,6 +662,11 @@ private:
             type->doDelete();
         }
     }
+    void cleanupType(const RtlTypeInfo *type)
+    {
+        if (type)
+            type->doCleanup();
+    }
     const RtlTypeInfo *lookupType(const char *name, IPropertyTree *all)
     {
         const RtlTypeInfo ** found = types.getValue(name);
@@ -675,7 +707,8 @@ private:
         const char *child = type->queryProp("child");
         if (child)
             info.childType = lookupType(child, all);
-        if ((info.fieldType & RFTMkind) == type_record)
+        unsigned baseType = (info.fieldType & RFTMkind);
+        if ((baseType == type_record) || (baseType == type_ifblock))
         {
             unsigned numFields = type->getCount("fields");
             info.fieldsArray = new const RtlFieldInfo * [numFields+1];
@@ -700,7 +733,17 @@ private:
                 n++;
             }
         }
-        return info.createRtlTypeInfo(callback);
+        if (baseType == type_ifblock)
+        {
+            unsigned fieldId = type->getPropInt("filterField");
+            const RtlTypeInfo * fieldType = lookupType(type->queryProp("filterType"), all);
+            info.filter = deserializeFieldFilter(fieldId, *fieldType, type->queryProp("filter"));
+        }
+
+        const RtlTypeInfo * result = info.createRtlTypeInfo(callback);
+        if (baseType == type_record)
+            patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
+        return result;
     }
 
     const RtlTypeInfo *deserializeType(MemoryBuffer &type)
@@ -720,6 +763,18 @@ private:
             type.readPacked(childIdx);
             info.childType = lookupType(childIdx);
         }
+
+        unsigned baseType = (info.fieldType & RFTMkind);
+        if (baseType == type_ifblock)
+        {
+            unsigned fieldId;
+            type.readPacked(fieldId);
+            unsigned childIdx;
+            type.readPacked(childIdx);
+            const RtlTypeInfo * fieldType = lookupType(childIdx);
+            info.filter = deserializeFieldFilter(fieldId, *fieldType, type);
+        }
+
         if (info.fieldType & RFTMhasFields)
         {
             unsigned numFields;
@@ -730,6 +785,8 @@ private:
             {
                 const char *fieldName;
                 type.read(fieldName);
+                if (fieldName[0] == '\0')
+                    fieldName = nullptr;
                 unsigned fieldType;
                 type.readPacked(fieldType);
                 unsigned fieldFlags;
@@ -750,7 +807,10 @@ private:
             }
         }
         info.fieldType &= ~RFTMserializerFlags;
-        return info.createRtlTypeInfo(callback);
+        const RtlTypeInfo * result = info.createRtlTypeInfo(callback);
+        if (baseType == type_record)
+            patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
+        return result;
     }
     void patchIndexFilePos()
     {
@@ -772,6 +832,25 @@ private:
             }
         }
     }
+    void patchIfBlockParentRow(const RtlTypeInfo * fieldType, const RtlRecordTypeInfo * parentRow)
+    {
+        const RtlFieldInfo * const * fields = fieldType->queryFields();
+        for (;*fields;fields++)
+        {
+            const RtlFieldInfo * cur = *fields;
+            if (!cur)
+                break;
+
+            const RtlTypeInfo * curType = cur->type;
+            if ((curType->fieldType & RFTMkind) == type_ifblock)
+            {
+                const RtlDynamicIfBlockTypeInfo * constifblock = static_cast<const RtlDynamicIfBlockTypeInfo *>(curType);
+                RtlDynamicIfBlockTypeInfo * ifblock = const_cast<RtlDynamicIfBlockTypeInfo *>(constifblock);
+                ifblock->setParent(parentRow);
+                patchIfBlockParentRow(curType, parentRow);
+            }
+        }
+    }
 };
 
 extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback)
@@ -801,7 +880,15 @@ extern ECLRTL_API bool dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
 extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __result, IOutputMetaData &  metaVal)
 {
     MemoryBuffer ret;
-    CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo());
+    try
+    {
+        CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo());
+    }
+    catch (IException * e)
+    {
+        ret.clear();
+        e->Release();
+    }
     __lenResult = ret.length();
     __result = ret.detach();
 }
@@ -809,19 +896,27 @@ extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __re
 extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &metaVal)
 {
     StringBuffer ret;
-    CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
+    try
+    {
+        CRtlFieldTypeSerializer::serialize(ret, metaVal.queryTypeInfo());
 
 #ifdef _DEBUG
-    StringBuffer ret2;
-    CRtlFieldTypeDeserializer deserializer(nullptr);
-    CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
-    assert(streq(ret, ret2));
-    MemoryBuffer out;
-    CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
-    CRtlFieldTypeDeserializer bindeserializer(nullptr);
-    CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
-    assert(streq(ret, ret2));
+        StringBuffer ret2;
+        CRtlFieldTypeDeserializer deserializer(nullptr);
+        CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
+        assert(streq(ret, ret2));
+        MemoryBuffer out;
+        CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
+        CRtlFieldTypeDeserializer bindeserializer(nullptr);
+        CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
+        assert(streq(ret, ret2));
 #endif
+    }
+    catch (IException * e)
+    {
+        e->errorMessage(ret.clear());
+        e->Release();
+    }
 
     __lenResult = ret.length();
     __result = ret.detach();
@@ -1494,7 +1589,7 @@ public:
     {
         for (unsigned idx = 0; idx < srcRecInfo.getNumFields(); idx++)
         {
-            unsigned matchIdx = destRecInfo.getFieldNum(destRecInfo.queryName(idx));
+            unsigned matchIdx = destRecInfo.getFieldNum(srcRecInfo.queryName(idx));
             if (matchIdx != -1)
             {
                 const RtlTypeInfo *srcType = srcRecInfo.queryType(idx);

rtl/eclrtl/rtldynfield.hpp

@@ -36,6 +36,7 @@ public:
     const char *className = nullptr;
     const RtlTypeInfo *childType = nullptr;
     const RtlFieldInfo * * fieldsArray = nullptr;
+    const IFieldFilter * filter = nullptr;
 
     const RtlTypeInfo *createRtlTypeInfo(IThorIndexCallback *_callback) const;
 };
@@ -79,14 +80,14 @@ interface IRtlFieldTypeDeserializer : public IInterface
      * @param key  A unique pointer used to dedup typeinfo structures
      * @return     RtlTypeInfo structure
      */
-    virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const ITypeInfo *key) = 0;
+    virtual const RtlTypeInfo *addType(FieldTypeInfoStruct &info, const IInterface *typeOrIfblock) = 0;
     /*
      * Check if a type has already been created for a given key
      *
      * @param key  A unique pointer used to dedup typeinfo structures
      * @return     RtlTypeInfo structure, or nullptr if not yet created
      */
-    virtual const RtlTypeInfo *lookupType(const ITypeInfo *key) const = 0;
+    virtual const RtlTypeInfo *lookupType(const IInterface *key) const = 0;
     /*
      * Create RtlFieldInfo structure as part of a RtlTypeInfo tree
      *

rtl/eclrtl/rtlfield.cpp

@@ -18,12 +18,15 @@
 #include "platform.h"
 #include <math.h>
 #include <stdio.h>
+#include <atomic>
 #include "jmisc.hpp"
 #include "jlib.hpp"
 #include "eclhelper.hpp"
 #include "eclrtl_imp.hpp"
 #include "rtlfield.hpp"
 #include "rtlds_imp.hpp"
+#include "rtlrecord.hpp"
+#include "rtlkey.hpp"
 #include "nbcd.hpp"
 
 static const char * queryXPath(const RtlFieldInfo * field)
@@ -253,6 +256,11 @@ const RtlTypeInfo * RtlTypeInfoBase::queryChildType() const
     return NULL; 
 }
 
+const IFieldFilter * RtlTypeInfoBase::queryFilter() const
+{
+    return nullptr;
+}
+
 size32_t RtlTypeInfoBase::deserialize(ARowBuilder & builder, IRowDeserializerSource & in, size32_t offset) const
 {
     size32_t thisSize = size(nullptr, nullptr);
@@ -2849,8 +2857,10 @@ size32_t RtlRecordTypeInfo::deserialize(ARowBuilder & builder, IRowDeserializerS
 
 void RtlRecordTypeInfo::readAhead(IRowPrefetcherSource & in) const
 {
-    //Will not generally be called because it will have been expanded
-    return readAheadFields(fields, in);
+    //This could be called if the record contains ifblocks
+    in.noteStartChild();
+    readAheadFields(fields, in);
+    in.noteFinishChild();
 }
 
 
@@ -3802,12 +3812,75 @@ unsigned RtlIfBlockTypeInfo::hash(const byte * self, unsigned inhash) const
     return inhash;
 }
 
-bool RtlDynamicIfBlockTypeInfo::getCondition(const byte * selfrow) const
+bool RtlComplexIfBlockTypeInfo::getCondition(const RtlRow & selfrow) const
 {
-#ifdef _DEBUG
-    // Temporary code to help my testing, until proper implementation available
-    return selfrow[3] != 2;
-#endif
+    //Call the function in the derived class that evaluates the test condition
+    return getCondition(selfrow.queryRow());
+}
+
+
+static CriticalSection ifcs;
+RtlSerialIfBlockTypeInfo::~RtlSerialIfBlockTypeInfo()
+{
+    ::Release(filter);
+    delete parentRecord;
+}
+
+void RtlSerialIfBlockTypeInfo::doCleanup() const
+{
+    delete parentRecord;
+    parentRecord = nullptr;
+    ::Release(filter);
+    filter = nullptr;
+    RtlIfBlockTypeInfo::doCleanup();
+}
+
+
+bool RtlSerialIfBlockTypeInfo::getCondition(const byte * selfrow) const
+{
+    const IFieldFilter * filter = resolveCondition();
+    RtlDynRow row(*parentRecord, nullptr);
+    //Can only expand offset for fields up to and including this if block - otherwise it will call getCondition() again...
+    row.setRow(selfrow, numPrevFields);
+    return filter->matches(row);
+}
+
+bool RtlSerialIfBlockTypeInfo::getCondition(const RtlRow & selfrow) const
+{
+    return resolveCondition()->matches(selfrow);
+}
+
+const IFieldFilter * RtlSerialIfBlockTypeInfo::queryFilter() const
+{
+    return resolveCondition();
+}
+
+const IFieldFilter * RtlSerialIfBlockTypeInfo::resolveCondition() const
+{
+    //The following cast is a hack to avoid <atomic> being included from the header
+    //if parentRecord is not initialised then may need to create the filter condition
+    typedef std::atomic<RtlRecord *> AtomicRtlRecord;
+    AtomicRtlRecord & atomicParentRecord = *reinterpret_cast<AtomicRtlRecord *>(&parentRecord);
+    RtlRecord * parent = atomicParentRecord.load(std::memory_order_relaxed);
+    if (!parent)
+    {
+        CriticalBlock block(ifcs);
+        parent = atomicParentRecord.load(std::memory_order_relaxed);
+        if (!parent)
+        {
+            if (!filter)
+                filter = createCondition();
+            parent = new RtlRecord(*rowType, true);
+            atomicParentRecord.store(parent, std::memory_order_relaxed);
+            numPrevFields = parentRecord->queryIfBlockLimit(this);
+        }
+    }
+    return filter;
+}
+
+IFieldFilter * RtlDynamicIfBlockTypeInfo::createCondition() const
+{
+    //The filter should be initialised on deserialization
     UNIMPLEMENTED;
 }
 

rtl/eclrtl/rtlfield.hpp

@@ -29,6 +29,8 @@ in that context.  For that reason the classes have no destructors.
 The file rtldynfield contains classes which manage instances of these classes which are dynamically created.
 */
 
+interface IFieldFilter;
+class RtlRow;
 size32_t ECLRTL_API getMinSize(const RtlFieldInfo * const * fields);
 
 // A base implementation of RtlTypeInfo
@@ -59,9 +61,11 @@ struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
     virtual const char * queryLocale() const override;
     virtual const RtlFieldInfo * const * queryFields() const override;
     virtual const RtlTypeInfo * queryChildType() const override;
+    virtual const IFieldFilter * queryFilter() const override;
 
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
     virtual void readAhead(IRowPrefetcherSource & in) const override;
+    virtual void doCleanup() const override {}
 protected:
     size32_t buildUtf8ViaString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
     void getUtf8ViaString(size32_t & resultLen, char * & result, const void * ptr) const;
@@ -577,10 +581,15 @@ struct ECLRTL_API RtlDictionaryTypeInfo : public RtlCompoundTypeInfo
 
 struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
 {
-    constexpr inline RtlIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlTypeInfoBase(_fieldType, _length), fields(_fields) {}
+    constexpr inline RtlIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields, const RtlRecordTypeInfo * _rowType)
+    : RtlTypeInfoBase(_fieldType, _length), fields(_fields), rowType(_rowType) {}
+
     const RtlFieldInfo * const * fields;                // null terminated
+    const RtlRecordTypeInfo * rowType;                  // may be updated when parent deserialized
 
     virtual bool getCondition(const byte * selfrow) const = 0;
+    virtual bool getCondition(const RtlRow & selfrow) const = 0;
+
     virtual size32_t getMinSize() const override;
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
@@ -599,11 +608,51 @@ struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
     virtual const RtlFieldInfo * const * queryFields() const override { return fields; }
 };
 
-struct ECLRTL_API RtlDynamicIfBlockTypeInfo final : public RtlIfBlockTypeInfo
+//Ifblock for complex conditions that cannot be serialized
+struct ECLRTL_API RtlComplexIfBlockTypeInfo : public RtlIfBlockTypeInfo
 {
-    constexpr inline RtlDynamicIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlIfBlockTypeInfo(_fieldType, _length, _fields) {}
+    constexpr inline RtlComplexIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields, const RtlRecordTypeInfo * _rowType)
+    : RtlIfBlockTypeInfo(_fieldType, _length, _fields, _rowType) {}
+
+    using RtlIfBlockTypeInfo::getCondition;
+    virtual bool getCondition(const RtlRow & selfrow) const override;
+};
+
+struct ECLRTL_API RtlSerialIfBlockTypeInfo : public RtlIfBlockTypeInfo
+{
+    constexpr inline RtlSerialIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields, const RtlRecordTypeInfo * _rowType)
+    : RtlIfBlockTypeInfo(_fieldType, _length, _fields, _rowType) {}
+    ~RtlSerialIfBlockTypeInfo();
+
     virtual bool getCondition(const byte * selfrow) const override;
+    virtual bool getCondition(const RtlRow & selfrow) const override;
+    virtual IFieldFilter * createCondition() const = 0;
+    virtual const IFieldFilter * queryFilter() const override;
+
+    virtual void doCleanup() const override;
+protected:
+    const IFieldFilter * resolveCondition() const;
+
+    //The following are initialised on demand in resolveCondition()
+    mutable const IFieldFilter * filter = nullptr;
+    mutable RtlRecord * parentRecord = nullptr;
+    mutable unsigned numPrevFields = 0;
+};
+
+struct ECLRTL_API RtlSimpleIfBlockTypeInfo : public RtlSerialIfBlockTypeInfo
+{
+    constexpr inline RtlSimpleIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields, const RtlRecordTypeInfo * _rowType)
+    : RtlSerialIfBlockTypeInfo(_fieldType, _length, _fields, _rowType) {}
+};
+
+struct ECLRTL_API RtlDynamicIfBlockTypeInfo final : public RtlSerialIfBlockTypeInfo
+{
+    inline RtlDynamicIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields, const RtlRecordTypeInfo * _rowType, const IFieldFilter * ownedFilter)
+    : RtlSerialIfBlockTypeInfo(_fieldType, _length, _fields, _rowType) { filter = ownedFilter; }
+    virtual IFieldFilter * createCondition() const override;
     virtual void doDelete() const final override { delete this; }
+
+    void setParent(const RtlRecordTypeInfo * _rowType) { rowType = _rowType; }
 };
 
 struct ECLRTL_API RtlBitfieldTypeInfo : public RtlTypeInfoBase

rtl/eclrtl/rtlkey.hpp

@@ -182,6 +182,8 @@ enum TransitionMask : byte
     CMPge = CMPgt | CMPeq,
     CMPminmask = CMPgt|CMPmin,
     CMPmaxmask = CMPlt|CMPmax,
+
+    CMPnovalue = 0x80,  // Used when serializing.
 };
 
 class ValueTransition;
@@ -207,7 +209,9 @@ interface IValueSet : public IInterface
     virtual void unionSet(const IValueSet *) = 0;
     virtual void excludeSet(const IValueSet *) = 0;
     virtual void intersectSet(const IValueSet *) = 0;
-    virtual StringBuffer & serialize(StringBuffer & out) const= 0;
+    virtual StringBuffer & serialize(StringBuffer & out) const = 0;
+    virtual MemoryBuffer & serialize(MemoryBuffer & out) const = 0;
+    virtual bool equals(const IValueSet & _other) const = 0;
 
 //following are primarily for use from the code generator
     virtual ValueTransition * createRawTransition(TransitionMask mask, const void * value) const = 0;
@@ -287,12 +291,16 @@ public:
 //Simple row matching
     virtual bool matches(const RtlRow & row) const = 0;
     virtual unsigned queryFieldIndex() const = 0;
+    virtual const RtlTypeInfo & queryType() const = 0;
     virtual int compareRow(const RtlRow & left, const RtlRow & right) const = 0;
     virtual int compareLowest(const RtlRow & left, unsigned range) const = 0;
     virtual int compareHighest(const RtlRow & left, unsigned range) const = 0;
     virtual bool isEmpty() const = 0;
     virtual bool isWild() const = 0;
 
+    virtual StringBuffer & serialize(StringBuffer & out) const = 0;
+    virtual MemoryBuffer & serialize(MemoryBuffer & out) const = 0;
+
     virtual unsigned numRanges() const = 0;
     virtual int findForwardMatchRange(const RtlRow & row, unsigned & matchRange) const = 0;
     virtual unsigned queryScore() const = 0;
@@ -305,6 +313,9 @@ extern ECLRTL_API IFieldFilter * createFieldFilter(unsigned fieldId, const RtlTy
 extern ECLRTL_API IFieldFilter * createFieldFilter(unsigned fieldId, IValueSet * values);
 extern ECLRTL_API IFieldFilter * createWildFieldFilter(unsigned fieldId, const RtlTypeInfo & type);
 
+extern ECLRTL_API IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type, const char * src);
+extern ECLRTL_API IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type, MemoryBuffer & in);
+
 
 
 #endif

rtl/eclrtl/rtlnewkey.cpp

@@ -1,7 +1,6 @@
 /*##############################################################################
 
 
-    Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
 
@@ -168,17 +167,54 @@ public:
             memcpy(value, _value, size);
         }
     }
+    ValueTransition(const RtlTypeInfo & type, MemoryBuffer & in)
+    {
+        byte inmask;
+        in.read(inmask);
+        if (!(inmask & CMPnovalue))
+        {
+            mask = (TransitionMask)inmask;
+            size32_t size = type.size(in.readDirect(0), nullptr);
+            value.allocateN(size, false);
+            in.read(size, value);
+        }
+        else
+        {
+            mask = (TransitionMask)(inmask & ~CMPnovalue);
+        }
+    }
     ValueTransition(TransitionMask _mask) : mask(_mask)
     {
         dbgassertex(isMaximum() || isMinimum());
     }
 
-    MemoryBuffer &serialize(const RtlTypeInfo & type, MemoryBuffer &mb) const
+    bool equals(const RtlTypeInfo & type, const ValueTransition & other) const
     {
-        mb.append((byte)mask);
-        size32_t size = type.size(value, nullptr);
-        memcpy(mb.reserve(size), value, size);
-        return mb;
+        if (mask != other.mask)
+            return false;
+        if (value && other.value)
+        {
+            return type.compare(value, other.value) == 0;
+        }
+        else
+            return !value && !other.value;
+    }
+
+    MemoryBuffer & serialize(const RtlTypeInfo & type, MemoryBuffer & out) const
+    {
+        byte outmask = mask;
+        if (value)
+        {
+            size32_t size = type.size(value, nullptr);
+            out.append(outmask);
+            out.append(size, value);
+        }
+        else
+        {
+            outmask |= CMPnovalue;
+            out.append(outmask);
+        }
+        return out;
     }
 
     int compareRaw(const RtlTypeInfo & type, const byte * field) const
@@ -336,6 +372,14 @@ public:
     {
     }
 
+    ValueSet(const RtlTypeInfo & _type, MemoryBuffer & in) : type(_type)
+    {
+        unsigned cnt;
+        in.readPacked(cnt);
+        for (unsigned i = 0; i < cnt; i++)
+            transitions.append(*new ValueTransition(type, in));
+    }
+
 // Methods for creating a value set
     virtual ValueTransition * createTransition(TransitionMask mask, unsigned __int64 value) const override
     {
@@ -743,6 +787,21 @@ public:
         killRange(lowerBound, upperBound);
     }
 
+    virtual bool equals(const IValueSet & _other) const override
+    {
+        const ValueSet & other = static_cast<const ValueSet &>(_other);
+        if (!type.equivalent(&other.type))
+            return false;
+        if (transitions.ordinality() != other.transitions.ordinality())
+            return false;
+        ForEachItemIn(i, transitions)
+        {
+            if (!transitions.item(i).equals(type, other.transitions.item(i)))
+                return false;
+        }
+        return true;
+    }
+
 // Methods for using a value set
     virtual bool isWild() const override;
     virtual unsigned numRanges() const override;
@@ -765,6 +824,14 @@ public:
         return describe(out);
     }
 
+    virtual MemoryBuffer & serialize(MemoryBuffer & out) const override
+    {
+        out.appendPacked((unsigned)transitions.ordinality());
+        ForEachItemIn(i, transitions)
+            transitions.item(i).serialize(type, out);
+        return out;
+    }
+
     virtual const RtlTypeInfo & queryType() const override
     {
         return type;
@@ -1000,6 +1067,7 @@ int ValueSet::findForwardMatchRange(const byte * field, unsigned & matchRange) c
 }
 
 IValueSet * createValueSet(const RtlTypeInfo & _type) { return new ValueSet(_type); }
+IValueSet * createValueSet(const RtlTypeInfo & _type, MemoryBuffer & in) { return new ValueSet(_type, in); }
 
 //---------------------------------------------------------------------------------------------------------------------
 
@@ -1022,6 +1090,11 @@ public:
         return field;
     }
 
+    virtual const RtlTypeInfo & queryType() const override
+    {
+        return type;
+    }
+
 protected:
     unsigned field;
     const RtlTypeInfo & type;
@@ -1048,6 +1121,8 @@ public:
 
     virtual unsigned queryScore() const override;
     virtual IFieldFilter *remap(unsigned newField) const override { return new SetFieldFilter(newField, values); }
+    virtual StringBuffer & serialize(StringBuffer & out) const override;
+    virtual MemoryBuffer & serialize(MemoryBuffer & out) const override;
 protected:
     Linked<IValueSet> values;
 };
@@ -1097,6 +1172,17 @@ unsigned SetFieldFilter::queryScore() const
     return score;
 }
 
+StringBuffer & SetFieldFilter::serialize(StringBuffer & out) const
+{
+    out.append('=');
+    return values->serialize(out);
+}
+MemoryBuffer & SetFieldFilter::serialize(MemoryBuffer & out) const
+{
+    out.append('=');
+    return values->serialize(out);
+}
+
 IFieldFilter * createFieldFilter(unsigned fieldId, IValueSet * values)
 {
     return new SetFieldFilter(fieldId, values);
@@ -1171,6 +1257,16 @@ public:
         return 0;
     }
     virtual IFieldFilter *remap(unsigned newField) const override { return new WildFieldFilter(newField, type); }
+
+    virtual StringBuffer & serialize(StringBuffer & out) const override
+    {
+        return out.append('*');
+    }
+
+    virtual MemoryBuffer & serialize(MemoryBuffer & out) const override
+    {
+        return out.append('*');
+    }
 };
 
 IFieldFilter * createWildFieldFilter(unsigned fieldId, const RtlTypeInfo & type)
@@ -1181,6 +1277,45 @@ IFieldFilter * createWildFieldFilter(unsigned fieldId, const RtlTypeInfo & type)
 
 //---------------------------------------------------------------------------------------------------------------------
 
+//Note, the fieldId could be serialized within the string, but it is needed to determine the type, and
+//passing it in allows this code to be decoupled from the type serialization code.
+IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type, const char * src)
+{
+    switch (*src)
+    {
+    case '*':
+        return createWildFieldFilter(fieldId, type);
+    case '=':
+        {
+            Owned<IValueSet> values = createValueSet(type);
+            deserializeSet(*values, src+1);
+            return createFieldFilter(fieldId, values);
+        }
+    }
+
+    UNIMPLEMENTED_X("Unknown Field Filter");
+}
+
+IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type, MemoryBuffer & in)
+{
+    char kind;
+    in.read(kind);
+    switch (kind)
+    {
+    case '*':
+        return createWildFieldFilter(fieldId, type);
+    case '=':
+        {
+            Owned<IValueSet> values = createValueSet(type, in);
+            return createFieldFilter(fieldId, values);
+        }
+    }
+
+    UNIMPLEMENTED_X("Unknown Field Filter");
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
 static int compareFieldFilters(IInterface * const * left, IInterface * const * right)
 {
     IFieldFilter * leftFilter = static_cast<IFieldFilter *>(*left);
@@ -1642,6 +1777,17 @@ protected:
         deserializeSet(*set, filter);
         checkSet(set, expected ? expected : filter);
 
+        MemoryBuffer mb;
+        set->serialize(mb);
+        Owned<IValueSet> newset = createValueSet(type, mb);
+        checkSet(newset, expected ? expected : filter);
+        CPPUNIT_ASSERT(set->equals(*newset));
+
+        StringBuffer s;
+        set->serialize(s);
+        Owned<IValueSet> newset2 = createValueSet(type);
+        deserializeSet(*newset2, s);
+        CPPUNIT_ASSERT(set->equals(*newset2));
     }
     void testUnion(RtlTypeInfo & type, const char * filter, const char * next, const char * expected)
     {

rtl/eclrtl/rtlrecord.cpp

@@ -112,8 +112,8 @@ static unsigned countFields(const RtlFieldInfo * const * fields, bool & contains
 class IfBlockInfo
 {
 public:
-    IfBlockInfo(const RtlFieldInfo &_field, const IfBlockInfo *_parent, unsigned _idx, unsigned _startIdx)
-    : field(_field), parent(_parent), idx(_idx), startIdx(_startIdx)
+    IfBlockInfo(const RtlFieldInfo &_field, const IfBlockInfo *_parent, unsigned _idx, unsigned _startIdx, unsigned _prevFields)
+    : field(_field), parent(_parent), idx(_idx), startIdx(_startIdx), prevFields(_prevFields)
     {
     }
 
@@ -132,11 +132,14 @@ public:
         return conditions[idx]==0;
     }
     inline unsigned queryStartField() const { return startIdx; }
+    inline unsigned numPrevFields() const { return prevFields; }
+    inline bool matchIfBlock(const RtlIfBlockTypeInfo * ifblock) const { return ifblock == field.type; }
 private:
     const RtlFieldInfo &field;
     const IfBlockInfo *parent = nullptr;  // for nested ifblocks
     unsigned idx;
     unsigned startIdx; // For ifblocks inside child records
+    unsigned prevFields; // number of fields before the if block
 };
 
 class RtlCondFieldStrInfo : public RtlFieldStrInfo
@@ -164,7 +167,7 @@ static unsigned expandNestedRows(unsigned idx, unsigned startIdx, const char *pr
             const IfBlockInfo *nestIfBlock = inIfBlock;
             if (isIfBlock)
             {
-                nestIfBlock = new IfBlockInfo(*cur, inIfBlock, ifblocks.ordinality(), startIdx);
+                nestIfBlock = new IfBlockInfo(*cur, inIfBlock, ifblocks.ordinality(), startIdx, idx);
                 ifblocks.append(nestIfBlock);
             }
             const RtlFieldInfo * const * nested = type->queryFields();
@@ -335,7 +338,8 @@ RtlRecord::~RtlRecord()
         {
             delete(ifblocks[i]);
         }
-        delete [] ifblocks;
+        //following as allocated as a ConstPointerArrayOf<IfBlockInfo>, rather than new []
+        free(ifblocks);
     }
     delete [] fixedOffsets;
     delete [] whichVariableOffset;
@@ -453,14 +457,22 @@ size32_t RtlRecord::deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource
 void RtlRecord::readAhead(IRowPrefetcherSource & in) const
 {
     // Note - this should not and can not be used when ifblocks present - canprefetch flag should take care of that.
-    dbgassertex(numIfBlocks==0);
-    for (unsigned i=0; i < numVarFields; i++)
+    if (numIfBlocks==0)
     {
-        unsigned fieldIndex = variableFieldIds[i];
-        in.skip(fixedOffsets[fieldIndex]);
-        queryType(fieldIndex)->readAhead(in);
+        for (unsigned i=0; i < numVarFields; i++)
+        {
+            unsigned fieldIndex = variableFieldIds[i];
+            in.skip(fixedOffsets[fieldIndex]);
+            queryType(fieldIndex)->readAhead(in);
+        }
+        in.skip(fixedOffsets[numFields]);
+    }
+    else
+    {
+        const RtlFieldInfo * const * field;
+        for (field = originalFields; *field; field++)
+            (*field)->type->readAhead(in);
     }
-    in.skip(fixedOffsets[numFields]);
 }
 
 int RtlRecord::compare(const byte * left, const byte * right) const
@@ -469,6 +481,16 @@ int RtlRecord::compare(const byte * left, const byte * right) const
     return compareFields(originalFields, left, right, false);
 }
 
+unsigned RtlRecord::queryIfBlockLimit(const RtlIfBlockTypeInfo * ifblock) const
+{
+    for (unsigned i=0; i < numIfBlocks; i++)
+    {
+        if (ifblocks[i]->matchIfBlock(ifblock))
+            return ifblocks[i]->numPrevFields();
+    }
+    throwUnexpected();
+}
+
 static const FieldNameToFieldNumMap *setupNameMap(const RtlRecord &record, std::atomic<const FieldNameToFieldNumMap *> &aNameMap)
 {
     const FieldNameToFieldNumMap *lnameMap = new FieldNameToFieldNumMap(record);

rtl/eclrtl/rtlrecord.hpp

@@ -223,6 +223,8 @@ public:
     void readAhead(IRowPrefetcherSource & in) const;
     int compare(const byte * left, const byte * right) const;
 
+    unsigned queryIfBlockLimit(const RtlIfBlockTypeInfo * ifblock) const;
+
     inline unsigned getNumFields() const { return numFields; }
     unsigned getNumKeyedFields() const;
     inline unsigned getNumVarFields() const { return numVarFields; }

rtl/include/eclhelper.hpp

@@ -64,6 +64,7 @@ interface IOutputMetaData;
 interface ICodeContext;
 interface IAtom;
 interface IException;
+interface IFieldFilter;
 class MemoryBuffer;
 class StringBuffer;
 class rtlRowBuilder;
@@ -381,6 +382,7 @@ interface RtlITypeInfo
     virtual const char * queryLocale() const = 0;
     virtual const RtlFieldInfo * const * queryFields() const = 0;               // null terminated list
     virtual const RtlTypeInfo * queryChildType() const = 0;
+    virtual const IFieldFilter * queryFilter() const = 0;
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const = 0;
     virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const = 0;
@@ -427,6 +429,7 @@ struct RtlTypeInfo : public RtlITypeInfo
     virtual bool canExtend(char &) const = 0;
     virtual bool canMemCmp() const = 0;
 
+    virtual void doCleanup() const = 0; // delete any special cached variables.
     virtual void doDelete() const = 0;  // Used in place of virtual destructor to allow constexpr constructors.
     virtual bool equivalent(const RtlTypeInfo *other) const = 0;
 public:

system/jlib/jexcept.cpp

@@ -1572,6 +1572,16 @@ IError *createError(IPropertyTree * tree)
 
 //---------------------------------------------------------------------------------------------------------------------
 
+void IErrorReceiver::ThrowStringException(int code,const char *format, ...) const
+{
+    va_list args;
+    va_start(args, format);
+    IException *ret = MakeStringExceptionVA(code, format, args);
+    va_end(args);
+    throw ret;
+}
+
+
 void IErrorReceiver::reportError(int errNo, const char *msg, const char *filename, int lineno, int column, int position)
 {
     Owned<IError> err = createError(errNo,msg,filename,lineno,column,position);

system/jlib/jexcept.hpp

@@ -232,6 +232,7 @@ interface jlib_decl IErrorReceiver : public IInterface
     virtual size32_t errCount() = 0;
     virtual size32_t warnCount() = 0;
     virtual void exportMappings(IWorkUnit * wu) const = 0;
+    virtual __declspec(noreturn) void ThrowStringException(int code,const char *format, ...) const __attribute__((format(printf, 3, 4), noreturn));            // override the global function to try and add more context information
 
     //global helper functions
     void reportError(int errNo, const char *msg, const char *filename, int lineno, int column, int pos);

testing/regress/ecl/key/nestedif.xml

@@ -4,3 +4,9 @@
  <Row><id>3</id><numnested>3</numnested><nestedrecord><Row><hasforename>true</hasforename><hassurname>false</hassurname><forename>Jim</forename></Row><Row><hasforename>false</hasforename><hassurname>true</hassurname><surname>Jones</surname></Row><Row><hasforename>false</hasforename><hassurname>false</hassurname></Row></nestedrecord></Row>
  <Row><id>99</id><numnested>0</numnested><nestedrecord></nestedrecord></Row>
 </Dataset>
+<Dataset name='Result 2'>
+ <Row><id>1</id><first><hasforename>true</hasforename><hassurname>true</hassurname><forename>Gavin</forename><surname>Hawthorn</surname></first><second><hasforename>false</hasforename><hassurname>false</hassurname></second></Row>
+ <Row><id>2</id><first><hasforename>false</hasforename><hassurname>false</hassurname></first><second><hasforename>true</hasforename><hassurname>false</hassurname><forename>X</forename></second></Row>
+ <Row><id>3</id><first><hasforename>true</hasforename><hassurname>false</hassurname><forename>Jim</forename></first><second><hasforename>false</hasforename><hassurname>true</hassurname><surname>Jones</surname></second></Row>
+ <Row><id>99</id><first><hasforename>false</hasforename><hassurname>false</hassurname></first><second><hasforename>false</hasforename><hassurname>false</hassurname></second></Row>
+</Dataset>

testing/regress/ecl/key/translatedisk.xml

@@ -0,0 +1,270 @@
+<Dataset name='Result 1'>
+ <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>0</filepos></Row>
+ <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>25</filepos></Row>
+ <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>50</filepos></Row>
+ <Row><dg_parentid>3</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>75</filepos></Row>
+ <Row><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>100</filepos></Row>
+ <Row><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>125</filepos></Row>
+ <Row><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>150</filepos></Row>
+ <Row><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>175</filepos></Row>
+ <Row><dg_parentid>8</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>200</filepos></Row>
+ <Row><dg_parentid>9</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>225</filepos></Row>
+ <Row><dg_parentid>10</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>250</filepos></Row>
+ <Row><dg_parentid>11</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>275</filepos></Row>
+ <Row><dg_parentid>12</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>300</filepos></Row>
+ <Row><dg_parentid>13</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>325</filepos></Row>
+ <Row><dg_parentid>14</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>350</filepos></Row>
+ <Row><dg_parentid>15</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>375</filepos></Row>
+ <Row><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>400</filepos></Row>
+ <Row><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>425</filepos></Row>
+ <Row><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>450</filepos></Row>
+ <Row><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>475</filepos></Row>
+ <Row><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>500</filepos></Row>
+ <Row><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>525</filepos></Row>
+ <Row><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>550</filepos></Row>
+ <Row><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>575</filepos></Row>
+ <Row><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>600</filepos></Row>
+ <Row><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>625</filepos></Row>
+ <Row><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>650</filepos></Row>
+ <Row><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>675</filepos></Row>
+ <Row><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>700</filepos></Row>
+ <Row><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>725</filepos></Row>
+ <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>750</filepos></Row>
+ <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>775</filepos></Row>
+ <Row><dg_parentid>32</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>800</filepos></Row>
+ <Row><dg_parentid>33</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>825</filepos></Row>
+ <Row><dg_parentid>34</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>850</filepos></Row>
+ <Row><dg_parentid>35</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>875</filepos></Row>
+ <Row><dg_parentid>36</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>900</filepos></Row>
+ <Row><dg_parentid>37</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>925</filepos></Row>
+ <Row><dg_parentid>38</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>950</filepos></Row>
+ <Row><dg_parentid>39</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>975</filepos></Row>
+ <Row><dg_parentid>40</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>1000</filepos></Row>
+ <Row><dg_parentid>41</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>1025</filepos></Row>
+ <Row><dg_parentid>42</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>1050</filepos></Row>
+ <Row><dg_parentid>43</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>1075</filepos></Row>
+ <Row><dg_parentid>44</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>1100</filepos></Row>
+ <Row><dg_parentid>45</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>1125</filepos></Row>
+ <Row><dg_parentid>46</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>1150</filepos></Row>
+ <Row><dg_parentid>47</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>1175</filepos></Row>
+ <Row><dg_parentid>48</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>1200</filepos></Row>
+ <Row><dg_parentid>49</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>1225</filepos></Row>
+ <Row><dg_parentid>50</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>1250</filepos></Row>
+ <Row><dg_parentid>51</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>1275</filepos></Row>
+ <Row><dg_parentid>52</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>1300</filepos></Row>
+ <Row><dg_parentid>53</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>1325</filepos></Row>
+ <Row><dg_parentid>54</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>1350</filepos></Row>
+ <Row><dg_parentid>55</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>1375</filepos></Row>
+ <Row><dg_parentid>56</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>1400</filepos></Row>
+ <Row><dg_parentid>57</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>1425</filepos></Row>
+ <Row><dg_parentid>58</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>1450</filepos></Row>
+ <Row><dg_parentid>59</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>1475</filepos></Row>
+ <Row><dg_parentid>60</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>1500</filepos></Row>
+ <Row><dg_parentid>61</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>1525</filepos></Row>
+ <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>1550</filepos></Row>
+ <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>1575</filepos></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>----</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>25</filepos></Row>
+ <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>50</filepos></Row>
+ <Row><dg_parentid>3</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>75</filepos></Row>
+ <Row><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>100</filepos></Row>
+ <Row><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>125</filepos></Row>
+ <Row><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>150</filepos></Row>
+ <Row><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>175</filepos></Row>
+ <Row><dg_parentid>8</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>200</filepos></Row>
+ <Row><dg_parentid>9</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>225</filepos></Row>
+ <Row><dg_parentid>10</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>250</filepos></Row>
+ <Row><dg_parentid>11</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>275</filepos></Row>
+ <Row><dg_parentid>12</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>300</filepos></Row>
+ <Row><dg_parentid>13</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>325</filepos></Row>
+ <Row><dg_parentid>14</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>350</filepos></Row>
+ <Row><dg_parentid>15</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>375</filepos></Row>
+ <Row><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>400</filepos></Row>
+ <Row><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>425</filepos></Row>
+ <Row><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>450</filepos></Row>
+ <Row><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>475</filepos></Row>
+ <Row><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>500</filepos></Row>
+ <Row><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>525</filepos></Row>
+ <Row><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>550</filepos></Row>
+ <Row><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>575</filepos></Row>
+ <Row><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>600</filepos></Row>
+ <Row><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>625</filepos></Row>
+ <Row><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>650</filepos></Row>
+ <Row><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>675</filepos></Row>
+ <Row><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>700</filepos></Row>
+ <Row><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>725</filepos></Row>
+ <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>750</filepos></Row>
+ <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>775</filepos></Row>
+ <Row><dg_parentid>32</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>800</filepos></Row>
+ <Row><dg_parentid>33</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>825</filepos></Row>
+ <Row><dg_parentid>34</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>850</filepos></Row>
+ <Row><dg_parentid>35</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>875</filepos></Row>
+ <Row><dg_parentid>36</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>900</filepos></Row>
+ <Row><dg_parentid>37</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>925</filepos></Row>
+ <Row><dg_parentid>38</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>950</filepos></Row>
+ <Row><dg_parentid>39</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>975</filepos></Row>
+ <Row><dg_parentid>40</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>1000</filepos></Row>
+ <Row><dg_parentid>41</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>1025</filepos></Row>
+ <Row><dg_parentid>42</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>1050</filepos></Row>
+ <Row><dg_parentid>43</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>1075</filepos></Row>
+ <Row><dg_parentid>44</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>1100</filepos></Row>
+ <Row><dg_parentid>45</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>1125</filepos></Row>
+ <Row><dg_parentid>46</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>1150</filepos></Row>
+ <Row><dg_parentid>47</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>1175</filepos></Row>
+ <Row><dg_parentid>48</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>1200</filepos></Row>
+ <Row><dg_parentid>49</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>1225</filepos></Row>
+ <Row><dg_parentid>50</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>1250</filepos></Row>
+ <Row><dg_parentid>51</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>1275</filepos></Row>
+ <Row><dg_parentid>52</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>1300</filepos></Row>
+ <Row><dg_parentid>53</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>1325</filepos></Row>
+ <Row><dg_parentid>54</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>1350</filepos></Row>
+ <Row><dg_parentid>55</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>1375</filepos></Row>
+ <Row><dg_parentid>56</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>1400</filepos></Row>
+ <Row><dg_parentid>57</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>1425</filepos></Row>
+ <Row><dg_parentid>58</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>1450</filepos></Row>
+ <Row><dg_parentid>59</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>1475</filepos></Row>
+ <Row><dg_parentid>60</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>1500</filepos></Row>
+ <Row><dg_parentid>61</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>1525</filepos></Row>
+ <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>1550</filepos></Row>
+ <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>1575</filepos></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>----</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>0</filepos></Row>
+ <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>25</filepos></Row>
+ <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>50</filepos></Row>
+ <Row><dg_parentid>3</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>75</filepos></Row>
+ <Row><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>100</filepos></Row>
+ <Row><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>125</filepos></Row>
+ <Row><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>150</filepos></Row>
+ <Row><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>175</filepos></Row>
+ <Row><dg_parentid>8</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>200</filepos></Row>
+ <Row><dg_parentid>9</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>225</filepos></Row>
+ <Row><dg_parentid>10</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>250</filepos></Row>
+ <Row><dg_parentid>11</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>275</filepos></Row>
+ <Row><dg_parentid>12</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>300</filepos></Row>
+ <Row><dg_parentid>13</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>325</filepos></Row>
+ <Row><dg_parentid>14</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>350</filepos></Row>
+ <Row><dg_parentid>15</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>375</filepos></Row>
+ <Row><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>400</filepos></Row>
+ <Row><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>425</filepos></Row>
+ <Row><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>450</filepos></Row>
+ <Row><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>475</filepos></Row>
+ <Row><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>500</filepos></Row>
+ <Row><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>525</filepos></Row>
+ <Row><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>550</filepos></Row>
+ <Row><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>575</filepos></Row>
+ <Row><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>600</filepos></Row>
+ <Row><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>625</filepos></Row>
+ <Row><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>650</filepos></Row>
+ <Row><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>675</filepos></Row>
+ <Row><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>700</filepos></Row>
+ <Row><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>725</filepos></Row>
+ <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>750</filepos></Row>
+ <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>775</filepos></Row>
+ <Row><dg_parentid>32</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>800</filepos></Row>
+ <Row><dg_parentid>33</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>825</filepos></Row>
+ <Row><dg_parentid>34</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>850</filepos></Row>
+ <Row><dg_parentid>35</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>875</filepos></Row>
+ <Row><dg_parentid>36</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>900</filepos></Row>
+ <Row><dg_parentid>37</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>925</filepos></Row>
+ <Row><dg_parentid>38</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>950</filepos></Row>
+ <Row><dg_parentid>39</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>975</filepos></Row>
+ <Row><dg_parentid>40</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1000</filepos></Row>
+ <Row><dg_parentid>41</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1025</filepos></Row>
+ <Row><dg_parentid>42</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1050</filepos></Row>
+ <Row><dg_parentid>43</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1075</filepos></Row>
+ <Row><dg_parentid>44</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1100</filepos></Row>
+ <Row><dg_parentid>45</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1125</filepos></Row>
+ <Row><dg_parentid>46</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1150</filepos></Row>
+ <Row><dg_parentid>47</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1175</filepos></Row>
+ <Row><dg_parentid>48</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1200</filepos></Row>
+ <Row><dg_parentid>49</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1225</filepos></Row>
+ <Row><dg_parentid>50</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1250</filepos></Row>
+ <Row><dg_parentid>51</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1275</filepos></Row>
+ <Row><dg_parentid>52</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1300</filepos></Row>
+ <Row><dg_parentid>53</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1325</filepos></Row>
+ <Row><dg_parentid>54</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1350</filepos></Row>
+ <Row><dg_parentid>55</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1375</filepos></Row>
+ <Row><dg_parentid>56</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1400</filepos></Row>
+ <Row><dg_parentid>57</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1425</filepos></Row>
+ <Row><dg_parentid>58</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1450</filepos></Row>
+ <Row><dg_parentid>59</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1475</filepos></Row>
+ <Row><dg_parentid>60</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1500</filepos></Row>
+ <Row><dg_parentid>61</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1525</filepos></Row>
+ <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1550</filepos></Row>
+ <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1575</filepos></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>0</filepos></Row>
+ <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>25</filepos></Row>
+ <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>50</filepos></Row>
+ <Row><dg_parentid>3</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>75</filepos></Row>
+ <Row><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>100</filepos></Row>
+ <Row><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>125</filepos></Row>
+ <Row><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>150</filepos></Row>
+ <Row><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>175</filepos></Row>
+ <Row><dg_parentid>8</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>200</filepos></Row>
+ <Row><dg_parentid>9</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>225</filepos></Row>
+ <Row><dg_parentid>10</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>250</filepos></Row>
+ <Row><dg_parentid>11</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>275</filepos></Row>
+ <Row><dg_parentid>12</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>300</filepos></Row>
+ <Row><dg_parentid>13</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>325</filepos></Row>
+ <Row><dg_parentid>14</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>350</filepos></Row>
+ <Row><dg_parentid>15</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>375</filepos></Row>
+ <Row><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>400</filepos></Row>
+ <Row><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>425</filepos></Row>
+ <Row><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>450</filepos></Row>
+ <Row><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>475</filepos></Row>
+ <Row><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>500</filepos></Row>
+ <Row><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>525</filepos></Row>
+ <Row><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>550</filepos></Row>
+ <Row><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>575</filepos></Row>
+ <Row><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>600</filepos></Row>
+ <Row><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>625</filepos></Row>
+ <Row><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>650</filepos></Row>
+ <Row><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>675</filepos></Row>
+ <Row><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>700</filepos></Row>
+ <Row><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>725</filepos></Row>
+ <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>750</filepos></Row>
+ <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>775</filepos></Row>
+ <Row><dg_parentid>32</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>800</filepos></Row>
+ <Row><dg_parentid>33</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>825</filepos></Row>
+ <Row><dg_parentid>34</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>850</filepos></Row>
+ <Row><dg_parentid>35</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>875</filepos></Row>
+ <Row><dg_parentid>36</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>900</filepos></Row>
+ <Row><dg_parentid>37</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>925</filepos></Row>
+ <Row><dg_parentid>38</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>950</filepos></Row>
+ <Row><dg_parentid>39</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>975</filepos></Row>
+ <Row><dg_parentid>40</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1000</filepos></Row>
+ <Row><dg_parentid>41</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1025</filepos></Row>
+ <Row><dg_parentid>42</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1050</filepos></Row>
+ <Row><dg_parentid>43</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1075</filepos></Row>
+ <Row><dg_parentid>44</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1100</filepos></Row>
+ <Row><dg_parentid>45</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1125</filepos></Row>
+ <Row><dg_parentid>46</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1150</filepos></Row>
+ <Row><dg_parentid>47</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1175</filepos></Row>
+ <Row><dg_parentid>48</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1200</filepos></Row>
+ <Row><dg_parentid>49</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1225</filepos></Row>
+ <Row><dg_parentid>50</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1250</filepos></Row>
+ <Row><dg_parentid>51</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1275</filepos></Row>
+ <Row><dg_parentid>52</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1300</filepos></Row>
+ <Row><dg_parentid>53</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1325</filepos></Row>
+ <Row><dg_parentid>54</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1350</filepos></Row>
+ <Row><dg_parentid>55</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1375</filepos></Row>
+ <Row><dg_parentid>56</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1400</filepos></Row>
+ <Row><dg_parentid>57</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1425</filepos></Row>
+ <Row><dg_parentid>58</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1450</filepos></Row>
+ <Row><dg_parentid>59</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1475</filepos></Row>
+ <Row><dg_parentid>60</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1500</filepos></Row>
+ <Row><dg_parentid>61</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1525</filepos></Row>
+ <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1550</filepos></Row>
+ <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1575</filepos></Row>
+</Dataset>

testing/regress/ecl/nestedif.ecl

@@ -52,3 +52,30 @@ saved4 := TABLE(saved3, { saved3, unsigned forceTransform := 0; }) : independent
 saved5 := TABLE(saved4, { saved4 } - [forceTransform]);
 
 output(saved5);
+
+
+//test that the same record type embedded twice in the same record resolves the child fields correctly
+
+baseRecord := RECORD
+    UNSIGNED id;
+    nestedRecord first;
+    nestedRecord second;
+END;
+
+baseTable := dataset([
+        {1, {true, true, 'Gavin', 'Hawthorn'}, { false, false } },
+        {2, {false, false}, { true, false, 'X' } },
+        {3, {true, false, 'Jim'}, { false, true, 'Jones' } },
+        {99, {false, false}, {false, false} }], baseRecord);
+
+bsaved1 := baseTable : independent(many);
+
+bsaved2 := bsaved1 : independent(few);
+
+bsaved3 := bsaved2 : independent(many);
+
+bsaved4 := TABLE(bsaved3, { bsaved3, unsigned forceTransform := 0; }) : independent(few);
+
+bsaved5 := TABLE(bsaved4, { bsaved4 } - [forceTransform]);
+
+output(bsaved5);

testing/regress/ecl/serializetypes.ecl

@@ -67,3 +67,41 @@ nfd := s.serializeRecordTypeNF(d[1]);  // not folded
 fd;
 nfd;
 fd = nfd;
+
+nestedRecord := RECORD
+    boolean hasForename;
+    unsigned surnameCount;
+    IFBLOCK (SELF.hasForename)
+        STRING forename;
+    END;
+    IFBLOCK (SELF.surnameCount != 0)
+        STRING surname;
+    END;
+END;
+
+mainRecord := RECORD
+    UNSIGNED id;
+    nestedRecord first;
+    nestedRecord second;
+    DATASET(nestedRecord) nestedRecord;
+//    DATASET(nestedRecord, COUNT(SELF.numNested)) nestedRecord;
+    IFBLOCK(SELF.id in [1,3,5,7,12])
+         STRING extra;
+    END;
+END;
+
+dx := dataset([], mainRecord);
+
+fx := s.dumpRecordType(dx[1]);     // folded
+nfx := s.dumpRecordTypeNF(dx[1]);  // not folded
+
+fx;
+nfx;
+fx = nfx; // currently false because Folded generating of meta for ifblocks is not yet implemented.
+
+fdx := s.serializeRecordType(dx[1]);     // folded
+nfdx := s.serializeRecordTypeNF(dx[1]);  // not folded
+
+fdx;
+nfdx;
+fdx = nfdx;

testing/regress/ecl/toxml.ecl

@@ -18,7 +18,7 @@
 //UseStandardFiles
 //version forceLayoutTranslation=0
 //version forceLayoutTranslation=1
-// it would be nice to also add version forceLayoutTranslation=2, but can't until ifblock condition metadata serialization is done properly
+//version forceLayoutTranslation=2
 
 import $.setup;
 prefix := setup.Files(false, false).FilePrefix;

testing/regress/ecl/translatedisk.ecl

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nothor
+//nohthor
+//version multiPart=false
+//version multiPart=true
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, false);
+
+//--- end of version configuration ---
+
+import $.Setup;
+
+boolean useLocal := false;
+Files := Setup.Files(multiPart, useLocal);
+
+
+DG_FlatFile := PRELOAD(DATASET(Files.DG_FileOut+'FLAT',{Files.DG_OutRec, UNSIGNED8 filepos{virtual(fileposition)}},FLAT));
+output(DG_FlatFile);
+OUTPUT('----');
+DG_FlatFile_add1 := PRELOAD(DATASET(Files.DG_FileOut+'FLAT',{Files.DG_OutRec,STRING newfield { default('new')}, UNSIGNED8 filepos{virtual(fileposition)}},FLAT));
+output(DG_FlatFile_add1);
+OUTPUT('----');
+
+newsub := RECORD
+  string2 newsub { default(':)') };
+END;
+
+DG_FlatFile_add2 := PRELOAD(DATASET(Files.DG_FileOut+'FLAT',{Files.DG_OutRec, STRING3 newfield { default('NEW')}, DATASET(newsub) sub, UNSIGNED8 filepos{virtual(fileposition)}},FLAT));
+output(DG_FlatFile_add2);
+
+d := table(DG_FlatFile, {unsigned8 fpos := DG_FlatFile.filepos} );
+output(FETCH(DG_FlatFile_add2, d, right.fpos, TRANSFORM (RECORDOF(DG_FlatFile_add2), SELF.filepos := right.fpos; SELF := LEFT)));

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -25,7 +25,7 @@
 #include "ctfile.hpp"
 #include "eclrtl.hpp"
 #include "thorfile.hpp"
-#include "rtldynfield.hpp"
+#include "thorfile.hpp"
 
 class IndexWriteActivityMaster : public CMasterActivity
 {
@@ -180,12 +180,7 @@ public:
             rtlFree(layoutMetaBuff);
         }
         // New record layout info
-        if (helper->queryDiskRecordSize()->queryTypeInfo())
-        {
-            MemoryBuffer out;
-            if (dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo()))
-                props.setPropBin("_rtlType", out.length(), out.toByteArray());
-        }
+        setRtlFormat(props, helper->queryDiskRecordSize());
         mpTag = container.queryJob().allocateMPTag();
         mpTag2 = container.queryJob().allocateMPTag();
     }

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -28,7 +28,7 @@
 #include "keybuild.hpp"
 #include "thbufdef.hpp"
 #include "backup.hpp"
-#include "rtldynfield.hpp"
+#include "thorfile.hpp"
 
 #define SINGLEPART_KEY_TRANSFER_SIZE 0x10000
 #define FEWWARNCAP 10
@@ -206,12 +206,7 @@ public:
         if(!metadata) metadata.setown(createPTree("metadata"));
         metadata->setProp("_record_ECL", helper->queryRecordECL());
 
-        if (helper->queryDiskRecordSize()->queryTypeInfo())
-        {
-            MemoryBuffer out;
-            if (dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo()))
-                metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
-        }
+        setRtlFormat(*metadata, helper->queryDiskRecordSize());
     }
 
     void close(IPartDescriptor &partDesc, unsigned &crc)

thorlcr/activities/thdiskbase.cpp

@@ -28,7 +28,6 @@
 
 #include "eclhelper.hpp" // tmp for IHThorArg interface
 #include "thdiskbase.ipp"
-#include "rtldynfield.hpp"
 
 CDiskReadMasterBase::CDiskReadMasterBase(CMasterGraphElement *info) : CMasterActivity(info), diskStats(info->queryJob(), diskReadRemoteStatistics)
 {
@@ -191,12 +190,7 @@ void CWriteMasterBase::init()
         const char *rececl= diskHelperBase->queryRecordECL();
         if (rececl&&*rececl)
             props.setProp("ECL", rececl);
-        if (diskHelperBase->queryDiskRecordSize()->queryTypeInfo())
-        {
-            MemoryBuffer out;
-            if (dumpTypeInfo(out, diskHelperBase->queryDiskRecordSize()->queryTypeInfo()))
-                props.setPropBin("_rtlType", out.length(), out.toByteArray());
-        }
+        setRtlFormat(props, diskHelperBase->queryDiskRecordSize());
 
         bool blockCompressed=false;
         void *ekey;