HPCC-18755 Remove translated segMonitor support

Translated segmonitors are not serializable and cannot be executed remotely
(because of the associated generated code).

The value of 'offset' passed as the second parameter to createTranslatedKeySegmentMonitor
was random. This implies that the 'docompare' and 'copy' (used by index search
code will not correctly, work, though matchesBuffer (which is used when
using this SegMonitor for filtering) is ok.

Because they can't actually work, it's safest to remove them and greatly
simplifies refactoring.

VarOffset segmonitors similarly will not currently work in in-memory indexes
as the code for comparing rows is not implemented. Once refactoring of
segmonitors is completed this should become moot and they will no longer be
needed. Again, best to remove, to simplify the refactoring.

Distinction between docompare and docompareraw is now moot. All
implementations were identical, so simplify the code.

Never generate wild segmonitors - they can be added by the runtime.

Remove test that requires VarOffset segmonitors, as they are (temporarily, I
hope) unsupported.

Fixed some lurking bugs revealed by making finish() do some actual work.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvidxsource.cpp

@@ -433,7 +433,7 @@ void IndexDataSource::applyFilter()
     {
         IStringSet & cur = values.item(i);
         bool extend = true; // almost certainly better
-        manager->append(createKeySegmentMonitor(extend, LINK(&cur), keyedOffsets.item(i), fieldSize(i)));
+        manager->append(createKeySegmentMonitor(extend, LINK(&cur), i, keyedOffsets.item(i), fieldSize(i)));
     }
     manager->finishSegmentMonitors();
 

common/remote/sockfile.cpp

@@ -2997,7 +2997,7 @@ class CRemoteKeyManager : public CSimpleInterfaceOf<IKeyManager>
         return count;
     }
 public:
-    CRemoteKeyManager(const char *_filename, unsigned _crc, IDelayedFile *_delayedFile) : filename(_filename), crc(_crc), delayedFile(_delayedFile)
+    CRemoteKeyManager(const char *_filename, unsigned _crc, IDelayedFile *_delayedFile) : segs(true), filename(_filename), crc(_crc), delayedFile(_delayedFile)
     {
     }
     ~CRemoteKeyManager()
@@ -4838,7 +4838,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
         mb.read(handle).read(keyName).read(keySize);
         if (segmentMonitors)
         {
-            SegMonitorList segs;
+            SegMonitorList segs(true);
             segs.deserialize(mb);
             return prepKey(handle, keyName, &segs);
         }

common/thorhelper/layouttrans.cpp

@@ -594,7 +594,7 @@ void CRecordLayoutTranslator::createDiskSegmentMonitors(SegmentMonitorContext co
             //fall through
 
         case FieldMapping::None:
-            monitor.setown(createWildKeySegmentMonitor(diskOffset, size));
+            monitor.setown(createWildKeySegmentMonitor(diskFieldNum, diskOffset, size));
             break;
 
         case FieldMapping::ChildDataset:
@@ -630,15 +630,10 @@ size32_t CRecordLayoutTranslator::transformRow(RowTransformContext * ctx, byte c
 void ExpandedSegmentMonitorList::append(IKeySegmentMonitor * monitor)
 {
     if(owner->failure) return;
-    while(monitor->getSize() > owner->activityKeySizes[monitors.ordinality()])
+    if(monitor->getSize() > owner->activityKeySizes[monitors.ordinality()])
     {
-        Owned<IKeySegmentMonitor> split = monitor->split(owner->activityKeySizes[monitors.ordinality()]);
-        if(!split)
-        {
-            owner->failure.setown(makeFailure(IRecordLayoutTranslator::Failure::UnsupportedFilter)->append("Unsupported filter (segment monitor) type (was larger than keyed field and unsplittable)"));
-            return;
-        }
-        monitors.append(*split.getLink());
+        owner->failure.setown(makeFailure(IRecordLayoutTranslator::Failure::UnsupportedFilter)->append("Unsupported filter (segment monitor) type (was larger than keyed field)"));
+        return;
     }
     if(monitor->getSize() < owner->activityKeySizes[monitors.ordinality()])
     {

ecl/hqlcpp/hqlcatom.cpp

@@ -92,7 +92,6 @@ IAtom * _spill_Atom;
 IAtom * _spillReason_Atom;
 IAtom * _steppedMeta_Atom;
 IAtom * subgraphAtom;
-IAtom * _translated_Atom;
 IAtom * userFunctionAtom;
 IAtom * utf8Atom;
 IAtom * wrapperAtom;
@@ -1501,6 +1500,5 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKESYSATOM(spill);
     MAKESYSATOM(spillReason);
     MAKESYSATOM(steppedMeta);
-    MAKESYSATOM(translated);
     return true;
 }

ecl/hqlcpp/hqlcatom.hpp

@@ -90,7 +90,6 @@ extern IAtom * _spill_Atom;
 extern IAtom * _spillReason_Atom;
 extern IAtom * _steppedMeta_Atom;
 extern IAtom * subgraphAtom;
-extern IAtom * _translated_Atom;
 extern IAtom * userFunctionAtom;
 extern IAtom * utf8Atom;
 extern IAtom * wrapperAtom;

ecl/hqlcpp/hqlsource.cpp

@@ -3703,10 +3703,9 @@ IHqlExpression * KeyConditionInfo::createConjunction()
     return result.getClear();
 }
 
-MonitorExtractor::MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _allowTranslatedConds) : translator(_translator) 
+MonitorExtractor::MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _isDiskRead) : translator(_translator)
 { 
     tableExpr = _tableExpr;
-    allowTranslatedConds = _allowTranslatedConds;
 
     if (_numKeyableFields <= 0)
     {
@@ -3721,7 +3720,7 @@ MonitorExtractor::MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator
     else
         numKeyableFields = (unsigned)_numKeyableFields;
 
-    onlyHozedCompares = !allowTranslatedConds;
+    onlyHozedCompares = !_isDiskRead;
 
     expandKeyableFields();
     cleanlyKeyedExplicitly = false;
@@ -3945,7 +3944,7 @@ bool MonitorExtractor::createGroupingMonitor(BuildCtx ctx, const char * listName
                 if (expr == &cur)
                 {
                     //MORE: Check the type of the field is legal.
-                    ctx.addQuotedF("%s->append(createWildKeySegmentMonitor(%u, %u));", listName, offset, curSize);
+                    ctx.addQuotedF("%s->append(createWildKeySegmentMonitor(%u, %u, %u));", listName, 0, offset, curSize);
                     maxOffset = offset+curSize;
                     return true;
                 }
@@ -4098,7 +4097,7 @@ void MonitorExtractor::buildKeySegmentInExpr(BuildMonitorState & buildState, Key
             OwnedHqlExpr test = createBoolExpr(no_eq, LINK(lhs), LINK(curValue));
             OwnedHqlExpr promoted = getExplicitlyPromotedCompare(test);
             OwnedHqlExpr compare, normalized;
-            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector, selectorInfo.isComputed);
+            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector);
             if (compare)
                 translator.buildFilter(subctx, compare);
 
@@ -4135,7 +4134,7 @@ void MonitorExtractor::buildKeySegmentInExpr(BuildMonitorState & buildState, Key
             OwnedHqlExpr test = createBoolExpr(no_eq, LINK(lhs), LINK(cur));
             OwnedHqlExpr promoted = getExplicitlyPromotedCompare(test);
             OwnedHqlExpr compare, normalized;
-            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector, selectorInfo.isComputed);
+            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector);
             if (compare)
                 translator.buildFilter(subctx, compare);
 
@@ -4158,36 +4157,31 @@ static IHqlExpression * createCompareRecast(node_operator op, IHqlExpression * v
 }
 
 
-void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isComputed)
+void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector)
 {
-    extractCompareInformation(ctx, expr->queryChild(0), expr->queryChild(1), compare, normalized, expandedSelector, isComputed);
+    extractCompareInformation(ctx, expr->queryChild(0), expr->queryChild(1), compare, normalized, expandedSelector);
 }
 
 
-void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isComputed)
+void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector)
 {
     LinkedHqlExpr compareValue = value->queryBody();
     OwnedHqlExpr recastValue;
-    if (isComputed)
-        normalized.setown(ensureExprType(compareValue, expandedSelector->queryType()));
-    else
+    if ((lhs->getOperator() != no_select) || (lhs->queryType() != compareValue->queryType()))
     {
-        if ((lhs->getOperator() != no_select) || (lhs->queryType() != compareValue->queryType()))
+        OwnedHqlExpr temp  = castToFieldAndBack(lhs, compareValue);
+        if (temp != compareValue)
         {
-            OwnedHqlExpr temp  = castToFieldAndBack(lhs, compareValue);
-            if (temp != compareValue)
-            {
-                //Force into a temporary variable since it will be used more than once, and reapply the field casting/
-                compareValue.setown(translator.buildSimplifyExpr(ctx, compareValue));
-                //cast to promoted type because sometimes evaluating can convert string to string<n>
-                Owned<ITypeInfo> promotedType = getPromotedECLType(lhs->queryType(), compareValue->queryType());
-                compareValue.setown(ensureExprType(compareValue, promotedType));
-                recastValue.setown(castToFieldAndBack(lhs, compareValue));
-            }
+            //Force into a temporary variable since it will be used more than once, and reapply the field casting/
+            compareValue.setown(translator.buildSimplifyExpr(ctx, compareValue));
+            //cast to promoted type because sometimes evaluating can convert string to string<n>
+            Owned<ITypeInfo> promotedType = getPromotedECLType(lhs->queryType(), compareValue->queryType());
+            compareValue.setown(ensureExprType(compareValue, promotedType));
+            recastValue.setown(castToFieldAndBack(lhs, compareValue));
         }
-
-        normalized.setown(invertTransforms(lhs, compareValue));
     }
+
+    normalized.setown(invertTransforms(lhs, compareValue));
     normalized.setown(foldHqlExpression(normalized));
 
     if (recastValue && recastValue != compareValue)
@@ -4221,7 +4215,7 @@ void MonitorExtractor::buildKeySegmentCompareExpr(BuildMonitorState & buildState
     OwnedHqlExpr compare;
     OwnedHqlExpr normalized;
     BuildCtx subctx(ctx);
-    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector, selectorInfo.isComputed);
+    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector);
     OwnedHqlExpr address = getMonitorValueAddress(subctx, normalized);
 
     HqlExprArray args;
@@ -4434,8 +4428,6 @@ void MonitorExtractor::buildKeySegmentExpr(BuildMonitorState & buildState, KeySe
                         OwnedHqlExpr targetVar = createVariable(targetSet, makeVoidType());
                         callAddAll(subctx, targetVar);
                     }
-                    else
-                        buildWildKeySegment(buildState, subctx, selectorInfo);
                     subctx.selectElse(ifStmt);
                 }
             }
@@ -4478,22 +4470,14 @@ void MonitorExtractor::buildKeySegmentExpr(BuildMonitorState & buildState, KeySe
 
     if (targetSet && !requiredSet)
     {
-        unsigned offset = (selectorInfo.isComputed || selectorInfo.mapOffset) ? 0 : selectorInfo.offset;
-        createMonitorText.appendf("createKeySegmentMonitor(%s, %s.getClear(), %u, %u)", 
-                                  boolToText(selectorInfo.keyedKind != KeyedYes), targetSet, offset, selectorInfo.size);
+        createMonitorText.appendf("createKeySegmentMonitor(%s, %s.getClear(), %u, %u, %u)",
+                                  boolToText(selectorInfo.keyedKind != KeyedYes), targetSet, selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size);
 
         buildState.popSetName();
     }
 
     if (createMonitorText.length())
-    {
-        if (selectorInfo.expandNeeded || selectorInfo.isComputed)
-            generateFormatWrapping(createMonitorText, selectorInfo.selector, selectorInfo.expandedSelector, buildState.curOffset);
-        else if (selectorInfo.mapOffset)
-            generateOffsetWrapping(createMonitorText, selectorInfo.selector);
-
         appendCtx->addQuotedF("%s->append(%s);", buildState.listName, createMonitorText.str());
-    }
 }
 
 
@@ -4529,85 +4513,13 @@ IHqlExpression * MonitorExtractor::getMonitorValueAddress(BuildCtx & ctx, IHqlEx
     return getPointer(bound.expr);
 }
 
-
-/*
-interface IKeySegmentOffsetTranslator : public IInterface
-{
-    virtual const void * getSegmentBase(const void * row) const = 0;
-};
-
-interface IKeySegmentFormatTranslator : public IInterface
-{
-    virtual void extractField(void * target, const void * row) const = 0;
-};
-*/
-
-void MonitorExtractor::generateOffsetWrapping(StringBuffer & createMonitorText, IHqlExpression * selector)
-{
-    unsigned curFieldIdx = getFieldNumber(tableExpr->queryNormalizedSelector(), selector);
-    StringBuffer s;
-    s.clear().append("createNewVarOffsetKeySegmentMonitor(").append(createMonitorText).append(",").append(curFieldIdx).append(")");
-    createMonitorText.swapWith(s);
-}
-
-void MonitorExtractor::generateFormatWrapping(StringBuffer & createMonitorText, IHqlExpression * selector, IHqlExpression * expandedSelector, unsigned curOffset)
-{
-    BuildCtx declarectx(*translator.code, declareAtom);
-    StringBuffer s, suffix, instanceName, className, factoryName;
-    unique_id_t id = translator.getUniqueId();
-    appendUniqueId(className.append("c"), id);
-    appendUniqueId(instanceName.append("i"), id);
-    appendUniqueId(factoryName.append("f"), id);
-
-    declarectx.setNextPriority(SegMonitorPrio);
-    BuildCtx classctx(declarectx);
-    //MORE: Use a base class for implementing this to save Link()/Release()
-    s.clear().append("struct ").append(className).append(" : public RtlCInterface, public IKeySegmentFormatTranslator");
-    suffix.append(instanceName).append(";");
-    classctx.addQuotedCompound(s, suffix);
-    classctx.addQuotedLiteral("virtual void Link() const override { RtlCInterface::Link(); }");
-    classctx.addQuotedLiteral("virtual bool Release() const override { return RtlCInterface::Release(); }");
-    classctx.addQuoted(s.clear().append("virtual const char * queryFactoryName() const override { return \"").append(factoryName).append("\"; }"));
-    classctx.addQuoted(s.clear().append("virtual unsigned queryHashCode() const override { return ").append(getExpressionCRC(selector)).append("; }"));
-
-    {
-        MemberFunction func(translator, classctx, "virtual void extractField(void * _target, const void * _row) const override");
-        classctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
-        func.ctx.addQuotedLiteral("const byte * row = (const byte *)_row;");
-        func.ctx.addQuotedLiteral("byte * target = (byte *)_target;");
-
-        OwnedHqlExpr castValue = ensureExprType(selector, expandedSelector->queryType());
-        LinkedHqlExpr targetField = expandedSelector->queryChild(1);
-        OwnedHqlExpr simpleRecord = createRecord(targetField);
-        OwnedHqlExpr targetDataset = createDataset(no_anon, LINK(simpleRecord));
-        OwnedHqlExpr target = createSelectExpr(LINK(targetDataset), LINK(targetField));
-
-        translator.bindTableCursor(func.ctx, tableExpr, "row");
-        translator.bindTableCursor(func.ctx, targetDataset, "target");
-        translator.buildAssign(func.ctx, target, castValue);
-    }
-
-    declarectx.setNextPriority(SegMonitorPrio);
-    declarectx.addQuoted(s.clear().append("IKeySegmentFormatTranslator * ").append(factoryName).append("() { return new ").append(className).append("; }"));
-    if (translator.spanMultipleCppFiles())
-    {
-        s.clear().append("extern IKeySegmentFormatTranslator * ").append(factoryName).append("();");
-        BuildCtx protoctx(*translator.code, mainprototypesAtom);
-        protoctx.addQuoted(s);
-    }
-
-    //Now generate the key segment monitor...
-    s.clear().append("createTranslatedKeySegmentMonitor(").append(createMonitorText).append(",").append(curOffset).append(",").append(factoryName).append("())");
-    createMonitorText.swapWith(s);
-}
-
 bool MonitorExtractor::buildSingleKeyMonitor(StringBuffer & createMonitorText, KeySelectorInfo & selectorInfo, BuildCtx & ctx, IHqlExpression & thisKey)
 {
     BuildCtx subctx(ctx);
     OwnedHqlExpr compare, normalized;
 
     StringBuffer funcName;
-    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector, selectorInfo.isComputed);
+    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector);
     if (compare)
         return false;
 
@@ -4638,10 +4550,9 @@ bool MonitorExtractor::buildSingleKeyMonitor(StringBuffer & createMonitorText, K
     StringBuffer addrText;
     translator.generateExprCpp(addrText, address);
 
-    unsigned offset = (selectorInfo.isComputed || selectorInfo.mapOffset) ? 0 : selectorInfo.offset;
     createMonitorText.append(funcName)
-                     .appendf("(%s, %u, %u, %s)", 
-                              boolToText(selectorInfo.keyedKind != KeyedYes), offset, selectorInfo.size, addrText.str());
+                     .appendf("(%s, %u, %u, %u, %s)",
+                              boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size, addrText.str());
     return true;
 }
 
@@ -4665,20 +4576,7 @@ KeyedKind getKeyedKind(HqlCppTranslator & translator, KeyConditionArray & matche
 void MonitorExtractor::buildEmptyKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo)
 {
     StringBuffer s;
-    ctx.addQuoted(s.appendf("%s->append(createEmptyKeySegmentMonitor(%s, %u, %u));", buildState.listName, boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.offset, selectorInfo.size));
-}
-
-
-void MonitorExtractor::buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned offset, unsigned size)
-{
-    StringBuffer s;
-    ctx.addQuoted(s.appendf("%s->append(createWildKeySegmentMonitor(%u, %u));", buildState.listName, offset, size));
-}
-
-
-void MonitorExtractor::buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo)
-{
-    buildWildKeySegment(buildState, ctx, selectorInfo.offset, selectorInfo.size);
+    ctx.addQuoted(s.appendf("%s->append(createEmptyKeySegmentMonitor(%s, %u, %u, %u));", buildState.listName, boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size));
 }
 
 
@@ -4733,7 +4631,9 @@ void MonitorExtractor::buildKeySegment(BuildMonitorState & buildState, BuildCtx
 
     StringBuffer s;
     KeyedKind keyedKind = getKeyedKind(translator, matches);
-    KeySelectorInfo selectorInfo(keyedKind, selector, expandedSelector, buildState.curOffset, curSize, (whichField >= firstOffsetField), false);
+    if (whichField >= firstOffsetField)
+        translator.throwError1(HQLERR_KeyedNotKeyed, getExprECL(field, s).str());
+    KeySelectorInfo selectorInfo(keyedKind, selector, expandedSelector, buildState.curFieldIdx, buildState.curOffset, curSize);
 
     bool ignoreKeyedExtend = false;
     if ((keyedKind == KeyedExtend) && buildState.wildPending() && !ignoreUnkeyed)
@@ -4758,10 +4658,7 @@ void MonitorExtractor::buildKeySegment(BuildMonitorState & buildState, BuildCtx
     if (matches.ordinality() && !ignoreKeyedExtend)
     {
         if (buildState.wildPending() && !ignoreUnkeyed)
-        {
-            buildWildKeySegment(buildState, ctx, buildState.wildOffset, buildState.curOffset-buildState.wildOffset);
             buildState.clearWild();
-        }
 
         HqlExprArray args;
         ForEachItemIn(i, matches)
@@ -4783,32 +4680,15 @@ void MonitorExtractor::buildKeySegment(BuildMonitorState & buildState, BuildCtx
         }
 
         if (buildState.wildPending() && noMergeSelects.contains(*selector))
-        {
-            buildWildKeySegment(buildState, ctx, buildState.wildOffset, buildState.curOffset-buildState.wildOffset);
             buildState.clearWild();
-        }
 
         if (!buildState.wildPending())
             buildState.wildOffset = buildState.curOffset;
     }
     buildState.curOffset += selectorInfo.size;
+    buildState.curFieldIdx++;
 }
 
-void MonitorExtractor::buildArbitaryKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned curSize, IHqlExpression * condition)
-{
-    IHqlExpression * left = condition->queryChild(0);
-    node_operator op = condition->getOperator();
-
-    StringBuffer createMonitorText;
-    OwnedHqlExpr field = createField(unknownId, getExpandedFieldType(left->queryType(), NULL), NULL);
-    OwnedHqlExpr pseudoSelector = createSelectExpr(getActiveTableSelector(), LINK(field));
-
-    KeySelectorInfo selectorInfo(KeyedExtend, left, pseudoSelector, buildState.curOffset, curSize, false, true);
-    BuildCtx subctx(ctx);
-    buildKeySegmentExpr(buildState, selectorInfo, subctx, NULL, *condition, MonitorFilterSkipAll);
-}
-
-
 void MonitorExtractor::spotSegmentCSE(BuildCtx & ctx)
 {
     //This could make things much better, but needs some thought
@@ -4901,25 +4781,6 @@ void MonitorExtractor::buildSegments(BuildCtx & ctx, const char * listName, bool
         }
     }
 
-    if (buildState.wildPending() && !ignoreUnkeyed)
-        buildWildKeySegment(buildState, ctx, buildState.wildOffset, buildState.curOffset-buildState.wildOffset);
-
-    //These really don't work very sensibly - we would need an offset that was constant, possibly based on the crc of the expression.
-    //I suspect they need a complete rethink.
-    ForEachItemIn(i, keyed.conditions)
-    {
-        KeyCondition & cur = keyed.conditions.item(i);
-        if (cur.selector->isAttribute() && cur.selector->queryName() == _translated_Atom)
-        {
-            BuildCtx subctx(ctx);
-            IHqlExpression * curExpr = cur.expr;
-            unsigned curSize = curExpr->queryChild(0)->queryType()->getSize();
-            buildArbitaryKeySegment(buildState, subctx, curSize, curExpr);
-            buildState.curOffset += curSize;
-            cur.generated = true;
-        }
-    }
-
     //check that all keyed entries have been matched
     ForEachItemIn(cond, keyed.conditions)
     {
@@ -4930,13 +4791,6 @@ void MonitorExtractor::buildSegments(BuildCtx & ctx, const char * listName, bool
 }
 
 
-static UniqueSequenceCounter translatedSequence;
-KeyCondition * MonitorExtractor::createTranslatedCondition(IHqlExpression * cond, KeyedKind keyedKind)
-{
-    OwnedHqlExpr seq = createSequence(no_attr, makeNullType(), _translated_Atom, translatedSequence.next());
-    return new KeyCondition(seq, cond, keyedKind);
-}
-
 bool MonitorExtractor::isKeySelect(IHqlExpression * select)
 {
     return (keyableSelects.find(*select) != NotFound);
@@ -5652,26 +5506,6 @@ bool MonitorExtractor::extractSimpleCompareFilter(KeyConditionInfo & matches, IH
         }
     }
 
-    if (!result && allowTranslatedConds)
-    {
-        duplicate = false;
-        ITypeInfo * type = l->queryType();
-        if (isKeyableType(type))
-        {
-            bool leftHasSelects = containsTableSelects(l);
-            bool rightHasSelects = containsTableSelects(r);
-            if (leftHasSelects && !rightHasSelects)
-            {
-                result.setown(createTranslatedCondition(expr, keyedKind));
-            }
-            else if (!leftHasSelects && rightHasSelects && (op != no_in) && (op != no_notin))
-            {
-                OwnedHqlExpr newFilter = createValue(getReverseOp(op), expr->getType(), LINK(r), LINK(l));
-                result.setown(createTranslatedCondition(newFilter, keyedKind));
-            }
-        }
-    }
-
     bool extracted = (result != NULL);
     if (extracted)
     {

ecl/hqlcpp/hqlsource.ipp

@@ -80,6 +80,7 @@ struct BuildMonitorState
     BuildMonitorState(BuildCtx & _funcctx, const char * _listName) : funcctx(_funcctx) 
     { 
         listName = _listName; 
+        curFieldIdx = 0;
         curOffset = 0;
         wildOffset = (unsigned) -1;
         numActiveSets = 0;
@@ -105,6 +106,7 @@ struct BuildMonitorState
     bool doneImplicitWarning;
     bool warnedAllConditionsWild;
     bool wildWasKeyed;
+    unsigned curFieldIdx;
     unsigned curOffset;
     unsigned wildOffset;
 };
@@ -114,32 +116,28 @@ enum MonitorFilterKind { NoMonitorFilter, MonitorFilterSkipEmpty, MonitorFilterS
 struct KeySelectorInfo
 {
 public:
-    KeySelectorInfo(KeyedKind _keyedKind, IHqlExpression * _selector, IHqlExpression * _expandedSelector, size32_t _offset, size32_t _size, bool _mapOffset, bool _isComputed)
+    KeySelectorInfo(KeyedKind _keyedKind, IHqlExpression * _selector, IHqlExpression * _expandedSelector, unsigned _fieldIdx, size32_t _offset, size32_t _size)
     {
         keyedKind = _keyedKind; 
         selector = _selector; 
-        expandedSelector = _expandedSelector; 
+        expandedSelector = _expandedSelector;
+        fieldIdx = _fieldIdx;
         offset = _offset; 
         size = _size;
-        expandNeeded = (selector->queryType() != expandedSelector->queryType());
-        mapOffset = _mapOffset;
-        isComputed = _isComputed;
     }
 
     IHqlExpression * selector;
     IHqlExpression * expandedSelector;
+    unsigned fieldIdx;
     size32_t offset;
     size32_t size;
     KeyedKind keyedKind;
-    bool expandNeeded;
-    bool mapOffset;
-    bool isComputed;
 };
 
 class MonitorExtractor
 {
 public:
-    MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _allowTranslatedConds);
+    MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool isDiskRead);
 
     void appendFilter(IHqlExpression * expr)                { keyed.appendPostFilter(expr); }
     void buildSegments(BuildCtx & ctx, const char * listName, bool _ignoreUnkeyed);
@@ -164,10 +162,7 @@ public:
     bool createGroupingMonitor(BuildCtx ctx, const char * listName, IHqlExpression * select, unsigned & maxOffset);
 
 protected:
-    void buildArbitaryKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned curSize, IHqlExpression * condition);
     void buildEmptyKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo);
-    void buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo);
-    void buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned offset, unsigned size);
     void buildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned whichField, unsigned curSize);
     void buildKeySegmentExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * target, IHqlExpression & thisKey, MonitorFilterKind filterKind);
     void buildKeySegmentCompareExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * requiredSet, IHqlExpression & thisKey);
@@ -187,8 +182,6 @@ protected:
     void expandKeyableFields();
     void expandSelects(IHqlExpression * expr, IHqlSimpleScope * expandedScope, IHqlExpression * keySelector, IHqlExpression * expandedSelector);;
     bool extractOrFilter(KeyConditionInfo & matches, IHqlExpression * filter, KeyedKind keyedKind);
-    void generateFormatWrapping(StringBuffer & createMonitorText, IHqlExpression * selector, IHqlExpression * expandedSelector, unsigned curOffset);
-    void generateOffsetWrapping(StringBuffer & createMonitorText, IHqlExpression * selector);
     IHqlExpression * getMonitorValueAddress(BuildCtx & ctx, IHqlExpression * value);
     IHqlExpression * getRangeLimit(ITypeInfo * fieldType, IHqlExpression * lengthExpr, IHqlExpression * value, int whichBoundary);
     IHqlExpression * invertTransforms(IHqlExpression * left, IHqlExpression * right);
@@ -201,8 +194,8 @@ protected:
     bool okToKey(IHqlExpression * select, KeyedKind keyedKind);
     IHqlExpression * queryKeyableSelector(IHqlExpression * expr);
     IHqlExpression * querySimpleJoinValue(IHqlExpression * field);
-    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isTranslated);
-    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isTranslated);
+    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
+    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
     IHqlExpression * unwindConjunction(HqlExprArray & matches, IHqlExpression * expr);
 
 protected:
@@ -238,7 +231,6 @@ protected:
     HqlExprCopyArray noMergeSelects;    // don't merge these fields (even for wildcards) because they are separate stepping fields.
     unsigned firstOffsetField;          // first field where the keyed offset is adjusted
     bool onlyHozedCompares;
-    bool allowTranslatedConds;
     bool ignoreUnkeyed;
     bool cleanlyKeyedExplicitly;
     bool keyedExplicitly;

roxie/ccd/ccdkey.cpp

@@ -269,11 +269,13 @@ public:
         ForEach(*fields)
         {
             IPropertyTree &field = fields->query();
+            unsigned fieldNum = field.getPropInt("@fieldNum", -1);
+            assertex(fieldNum != (unsigned) -1);
             unsigned offset = field.getPropInt("@offset", 0);
             unsigned size = field.getPropInt("@size", 0);
             bool isSigned = field.getPropBool("@isSigned", false);
             bool isLittleEndian = field.getPropBool("@isLittleEndian", false);
-            append(createDummyKeySegmentMonitor(offset, size, isSigned, isLittleEndian));
+            append(createDummyKeySegmentMonitor(fieldNum, offset, size, isSigned, isLittleEndian));
         }
     }
 
@@ -326,7 +328,7 @@ public:
 #endif
         ForEachItemIn(idx, segMonitors)
         {
-            int ret = segMonitors.item(idx).docompareraw(l, r);
+            int ret = segMonitors.item(idx).docompare(l, r);
             if (ret)
                 return ret;
         }
@@ -386,7 +388,7 @@ public:
         ForEachItemIn(idx, segMonitors)
         {
             IKeySegmentMonitor &item = segMonitors.item(idx);
-            ret.pad(indent+1).appendf("<Field offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", item.getOffset(), item.getSize(), item.isSigned(), item.isLittleEndian());
+            ret.pad(indent+1).appendf("<Field fieldNum='%d' offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", item.getFieldIdx(), item.getOffset(), item.getSize(), item.isSigned(), item.isLittleEndian());
         }
         ret.pad(indent).append("</FieldSet>\n");
         return ret;
@@ -979,8 +981,7 @@ class InMemoryIndexManager : implements IInMemoryIndexManager, public CInterface
             ForEachItemIn(idx, m)
             {
                 IKeySegmentMonitor &seg = m.item(idx);
-                xml.appendf("<Field offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", seg.getOffset(), seg.getSize(), seg.isSigned(), seg.isLittleEndian());
-//              xml.appendf("<Field offset='%d' size='%d' flags='%d'/>\n", seg.getOffset(), seg.getSize(), seg.getFlags());
+                xml.appendf("<Field fieldNum='%d' offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", seg.getFieldIdx(), seg.getOffset(), seg.getSize(), seg.isSigned(), seg.isLittleEndian());
             }
             xml.append("</FieldSet>\n");
         }
@@ -1307,11 +1308,12 @@ public:
             {
                 // MORE - we could share them, then comparison would be faster (pointer compare)
                 IKeySegmentMonitor &seg = perfect.item(idx);
+                unsigned fieldNum = seg.getFieldIdx();
                 unsigned offset = seg.getOffset();
                 unsigned size = seg.getSize();
                 bool isSigned = seg.isSigned();
                 bool isLittleEndian = seg.isLittleEndian();
-                newEntry->append(*createDummyKeySegmentMonitor(offset, size, isSigned, isLittleEndian));
+                newEntry->append(*createDummyKeySegmentMonitor(fieldNum, offset, size, isSigned, isLittleEndian));
             }
             tracked[a] = newEntry;
             hits[a] = noteHits ? noteHits : 1;
@@ -2054,14 +2056,14 @@ protected:
         di.load(testarray, sizeof(testarray), &x, 1);
         unsigned searchval = 1;
         InMemoryIndex order;
-        order.append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        order.append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         di.setBuilding();
         di.resort(order);
         InMemoryIndexManager indexes(false, "test1");
         indexes.append(*LINK(&di));
 
         Owned<IInMemoryIndexCursor> d = indexes.createCursor(dummy);
-        d->append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        d->append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         d->selectKey();
         d->reset();
         ASSERT(d->nextMatch()==&testarray[0]);
@@ -2070,7 +2072,7 @@ protected:
 
         d.setown(indexes.createCursor(dummy));
         searchval = 10;
-        d->append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        d->append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         d->selectKey();
         d->reset();
         ASSERT(d->nextMatch()==NULL);
@@ -2078,7 +2080,7 @@ protected:
         d.clear();
         di.deprecate();
 
-        Owned<IPropertyTree> keyInfo = createPTreeFromXMLString("<R><FieldSet><Field isLittleEndian='1' isSigned='0' offset='0' size='4'/><Field isLittleEndian='1' isSigned='1' offset='0' size='4'/></FieldSet></R>");
+        Owned<IPropertyTree> keyInfo = createPTreeFromXMLString("<R><FieldSet><Field isLittleEndian='1' isSigned='0' offset='0' fieldNum='0' size='4'/><Field isLittleEndian='1' isSigned='1' offset='0' fieldNum='0' size='4'/></FieldSet></R>");
         indexes.setKeyInfo(*keyInfo.get());
         Sleep(1000); // to give key time to build!
 
@@ -2088,7 +2090,7 @@ protected:
         set->addRange(&searchval, &searchval);
         searchval = 9;
         set->addRange(&searchval, &searchval);
-        d->append(createKeySegmentMonitor(false, set, 0, sizeof(unsigned)));
+        d->append(createKeySegmentMonitor(false, set, 0, 0, sizeof(unsigned)));
         d->selectKey();
         d->reset();
         ASSERT(*(int *) d->nextMatch()==2);
@@ -2108,19 +2110,19 @@ protected:
         const RtlFieldInfo * const fields [] = {&f1, nullptr};
         RtlRecord dummy(fields, true); // NOTE - not accurate but good enough for these tests
 
-        IKeySegmentMonitor *ksm = createSingleLittleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval);
+        IKeySegmentMonitor *ksm = createSingleLittleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval);
         ASSERT(ksm->isSigned()==false);
         ASSERT(ksm->isLittleEndian()==true);
         di.append(ksm);
-        di.append(createSingleKeySegmentMonitor(false, 4, sizeof(unsigned), &searchval));
-        di.append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        di.append(createSingleKeySegmentMonitor(false, 1, 4, sizeof(unsigned), &searchval));
+        di.append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         di.setBuilding();
         di.undeprecate();
 
         InMemoryIndex di2;
-        di2.append(createSingleKeySegmentMonitor(false, 12, sizeof(unsigned), &searchval));
-        di2.append(createSingleKeySegmentMonitor(false, 16, sizeof(unsigned), &searchval));
-        di2.append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        di2.append(createSingleKeySegmentMonitor(false, 3, 12, sizeof(unsigned), &searchval));
+        di2.append(createSingleKeySegmentMonitor(false, 4, 16, sizeof(unsigned), &searchval));
+        di2.append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         di2.setBuilding();
         di2.undeprecate();
         
@@ -2131,8 +2133,8 @@ protected:
         Owned<IInMemoryIndexCursor> dd = indexes.createCursor(dummy);
         InMemoryIndexCursor *d = QUERYINTERFACE(dd.get(), InMemoryIndexCursor);
         ASSERT(d != nullptr);
-        dd->append(createSingleKeySegmentMonitor(false, 4, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 1, 4, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval));
         ASSERT(d->postFilter.length()==2);
         ASSERT(d->segMonitors.length()==0);
         ASSERT(d->postFilter.item(0).getOffset()==4);
@@ -2146,8 +2148,8 @@ protected:
         dd.setown(indexes.createCursor(dummy));
         d = QUERYINTERFACE(dd.get(), InMemoryIndexCursor);
         ASSERT(d != nullptr);
-        dd->append(createSingleKeySegmentMonitor(false, 16, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 4, 16, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval));
         ASSERT(d->postFilter.length()==2);
         ASSERT(d->segMonitors.length()==0);
         ASSERT(d->postFilter.item(0).getOffset()==8);
@@ -2161,9 +2163,9 @@ protected:
         dd.setown(indexes.createCursor(dummy));
         d = QUERYINTERFACE(dd.get(), InMemoryIndexCursor);
         ASSERT(d != nullptr);
-        dd->append(createSingleKeySegmentMonitor(false, 12, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 16, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 3, 12, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 4, 16, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval));
         ASSERT(d->postFilter.length()==3);
         ASSERT(d->segMonitors.length()==0);
         ASSERT(d->postFilter.item(0).getOffset()==8);
@@ -2183,48 +2185,48 @@ protected:
 
         SegMonitorArray q;
         unsigned char searchval;
-        q.append(*createSingleKeySegmentMonitor(false, 'p', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'q', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'p', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'q', 1, &searchval));
         unsigned i;
         for (i = 0; i < 104; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'a', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'b', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'a', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'b', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 103; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'c', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'd', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'c', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'd', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 102; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'e', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'f', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'e', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'f', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 101; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'g', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'h', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'g', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'h', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 100; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'g', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'g', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'y', 1, &searchval));
         for (i = 0; i < 100; i++)
             stats.noteQuery(q, 0);
 

rtl/eclrtl/eclhelper_dyn.cpp

@@ -147,27 +147,27 @@ public:
             MemoryBuffer hibuffer;
             Owned<IStringSet> filterSet = createStringSet(fieldSize);
             deserializeSet(*filterSet, inrec.getMinRecordSize(), fieldType, filter);
-            aindex_t found = filterOffsets.bSearch(fieldOffset, compareOffsets);
-            if (found==NotFound)
+            while (filters.length()<=fieldNum)
             {
-                bool isNew;
-                unsigned insertPos = filterOffsets.bAdd(fieldOffset, compareOffsets, isNew);
-                dbgassertex(isNew);
-                filters.add(*filterSet.getClear(), insertPos);
-            }
-            else
-            {
-                filterSet.setown(filters.item(found).unionSet(filterSet));// Debatable - would intersect be more appropriate?
-                filters.replace(*filterSet.getClear(), found);
+                filters.append(nullptr);
+                filterOffsets.append(offsetCalculator.getOffset(filters.length()));
+                filterOffsets.append(offsetCalculator.getSize(filters.length()));
             }
+            IStringSet *prev = filters.item(fieldNum);
+            if (prev)
+                filterSet.setown(prev->unionSet(filterSet)); // Debatable - would intersect be more appropriate?
+            filters.replace(filterSet.getClear(), fieldNum);
+            filterOffsets.replace(fieldOffset, fieldOffset); // MORE - probably refactor this in  a bit
+            filterSizes.replace(fieldSize, fieldSize); // MORE - probably refactor this in  a bit
         }
     }
     void createSegmentMonitors(IIndexReadContext *irc)
     {
         ForEachItemIn(idx, filters)
         {
-            IStringSet &filter = filters.item(idx);
-            irc->append(createKeySegmentMonitor(false, LINK(&filter), filterOffsets.item(idx), filter.getSize()));
+            IStringSet *filter = filters.item(idx);
+            if (filter)
+                irc->append(createKeySegmentMonitor(false, LINK(filter), idx, filterOffsets.item(idx), filter->getSize()));
         }
     }
     void createSegmentMonitorsWithWild(IIndexReadContext *irc, unsigned keySize)
@@ -175,20 +175,27 @@ public:
         unsigned lastOffset = 0;
         ForEachItemIn(idx, filters)
         {
-            IStringSet &filter = filters.item(idx);
+            IStringSet *filter = filters.item(idx);
             unsigned offset =  filterOffsets.item(idx);
-            unsigned size = filter.getSize();
-            if (offset > lastOffset)
-                irc->append(createWildKeySegmentMonitor(lastOffset, offset-lastOffset));
-            irc->append(createKeySegmentMonitor(false, LINK(&filter), offset, size));
-            lastOffset = offset+size;
+            unsigned size =  filterSizes.item(idx);
+            if (filter)
+            {
+                assertex(size = filter->getSize());
+                irc->append(createKeySegmentMonitor(false, LINK(filter), idx, offset, size));
+            }
+            else
+                irc->append(createWildKeySegmentMonitor(idx, offset, size)); // MORE - move this logic to irc::append ?
         }
+        // MORE - trailing wild needs adding
+        /*
         if (keySize > lastOffset)
             irc->append(createWildKeySegmentMonitor(lastOffset, keySize-lastOffset));
+            */
     }
 protected:
-    IArrayOf<IStringSet> filters;
+    IPointerArrayOf<IStringSet> filters;
     UnsignedArray filterOffsets;
+    UnsignedArray filterSizes;
     const RtlRecord &inrec;
 };
 

rtl/eclrtl/rtlkey.hpp

@@ -28,9 +28,7 @@ enum KeySegmentMonitorSerializeType
     KSMST_SINGLEBIGSIGNEDKEYSEGMENTMONITOR,
     KSMST_SINGLELITTLESIGNEDKEYSEGMENTMONITOR,
     KSMST_CSINGLELITTLEKEYSEGMENTMONITOR,
-    KSMST_DUMMYKEYSEGMENTMONITOR,
     KSMST_OVERRIDEABLEKEYSEGMENTMONITOR,
-    KSMST_VAROFFSETKEYSEGMENTMONITOR,
     KSMST_max
 };
 
@@ -80,48 +78,39 @@ ECLRTL_API int memcmplittlesigned(const void *l, const void *r, unsigned size);
 
 class RtlRow;
 
-// In the parameter names below, "expanded" means it's a pointer to a "key buffer" value, while
-// "raw" means it's a pointer to a disk record. For indexes, the two are the same, but for
-// disk files where we may be using segmonitors as a way to serialize remote filters and to build on-the-fly indexes,
-// the distinction is significant.
-// To support varoffset segmonitors, all "raw" values are passed as const RtlRow *. We do not support varoffset segmonitors in
-// on-the-fly in-memory indexes, meaning that the functions taking "expanded" params do not need to support offset translation
-// and the varoffset segmonitor does not implement these functions.
-
 interface IKeySegmentMonitor : public IInterface
 {
 public:
     virtual bool increment(void * expandedRow) const = 0;
     virtual void setLow(void * expandedRow) const = 0;
+    virtual void setHigh(void * expandedRow) const = 0;
     virtual void endRange(void * expandedRow) const = 0;
+
     virtual bool matchesBuffer(const void * expandedRow) const = 0;
     virtual bool matches(const RtlRow * rawRow) const = 0;
-    virtual IKeySegmentMonitor *merge(IKeySegmentMonitor *with) const = 0;  // merge with adjacent, if possible
-    virtual IKeySegmentMonitor *combine(const IKeySegmentMonitor *with) const = 0; // combine with overlapping (compulsory)
-    virtual IKeySegmentMonitor * split(unsigned splitSize) = 0;
+
     virtual bool isWild() const = 0;
+    virtual unsigned getFieldIdx() const = 0;
     virtual unsigned getOffset() const = 0;
     virtual unsigned getSize() const = 0;
-    virtual void *queryValue() const = 0;
-    virtual int docompare(const void * expandedLeft, const void * rawRight) const = 0;
-//  virtual unsigned getFlags() const = 0;
     virtual bool isEmpty() const = 0;
     virtual bool equivalentTo(const IKeySegmentMonitor &other) const = 0;
     virtual bool isSigned() const = 0;
     virtual bool isLittleEndian() const = 0;
-    virtual int docompareraw(const void * rawLeft, const void * rawRight) const = 0; // NOTE - no RtlRow version since only used for in-memory index builds
+
+    virtual int docompare(const void * expandedLeft, const void * rawRight) const = 0;
     virtual unsigned queryHashCode() const = 0;
     virtual bool isWellKeyed() const = 0;
-    virtual bool setOffset(unsigned _offset) = 0;
     virtual bool isOptional() const = 0;
 
-    virtual void setHigh(void * expandedRow) const = 0;
     virtual bool isSimple() const = 0;
     virtual void copy(void *expandedRow, const void *rawRow) const = 0;
     virtual MemoryBuffer &serialize(MemoryBuffer &mb) const = 0;
     virtual KeySegmentMonitorSerializeType serializeType() const = 0;
     virtual IKeySegmentMonitor *clone() const = 0;
     virtual unsigned numFieldsRequired() const = 0;
+
+    virtual bool setOffset(unsigned _offset) = 0;  // Used by old record layout translator - to be removed at some point
 };
 
 interface IOverrideableKeySegmentMonitor  : public IKeySegmentMonitor
@@ -148,19 +137,6 @@ public:
     virtual void setMergeBarrier(unsigned offset) = 0;
 };
 
-interface IKeySegmentOffsetTranslator : public IInterface
-{
-    virtual const void * getSegmentBase(const void * row) const = 0;
-    virtual const char * queryFactoryName() const = 0;
-};
-
-interface IKeySegmentFormatTranslator : public IInterface
-{
-    virtual void extractField(void * target, const void * row) const = 0;
-    virtual const char * queryFactoryName() const = 0;
-    virtual unsigned queryHashCode() const = 0;
-};
-
 ECLRTL_API IStringSet *createRtlStringSet(size32_t size);
 ECLRTL_API IStringSet *createRtlStringSetEx(size32_t size, bool bigEndian, bool isSigned);
 ECLRTL_API IStringSet *createRtlStringValue(size32_t size, const char * value);
@@ -169,21 +145,18 @@ ECLRTL_API IStringSet *createRtlStringValue(size32_t size, const char * value);
 ECLRTL_API IStringSet *rtlUnionSet(IStringSet * lhs, IStringSet * rhs);
 ECLRTL_API IStringSet *rtlIntersectSet(IStringSet * lhs, IStringSet * rhs);
 
-ECLRTL_API IKeySegmentMonitor *createKeySegmentMonitor(bool optional, IStringSet *set, unsigned _offset, unsigned _size);
-ECLRTL_API IKeySegmentMonitor *createEmptyKeySegmentMonitor(bool optional, unsigned _offset, unsigned _size);
-ECLRTL_API IKeySegmentMonitor *createWildKeySegmentMonitor(unsigned _offset, unsigned _size);
-ECLRTL_API IKeySegmentMonitor *createDummyKeySegmentMonitor(unsigned _offset, unsigned _size, bool isSigned, bool isLittleEndian);
-ECLRTL_API IKeySegmentMonitor *createSingleKeySegmentMonitor(bool optional, unsigned _offset, unsigned _size, const void * value);
-ECLRTL_API IOverrideableKeySegmentMonitor *createOverrideableKeySegmentMonitor(IKeySegmentMonitor *base);
-ECLRTL_API IKeySegmentMonitor *createSingleBigSignedKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
-ECLRTL_API IKeySegmentMonitor *createSingleLittleSignedKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
-ECLRTL_API IKeySegmentMonitor *createSingleLittleKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
+// Various types of KeySegmentMonitor, called from generated code
 
-//takes over ownership of base
-ECLRTL_API IKeySegmentMonitor *createNewVarOffsetKeySegmentMonitor(IKeySegmentMonitor * base, unsigned fieldIdx);
+ECLRTL_API IKeySegmentMonitor *createKeySegmentMonitor(bool optional, IStringSet *set, unsigned _fieldIdx, unsigned _offset, unsigned _size);
+ECLRTL_API IKeySegmentMonitor *createEmptyKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned _offset, unsigned _size);
+ECLRTL_API IKeySegmentMonitor *createWildKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, unsigned _size);
+ECLRTL_API IKeySegmentMonitor *createDummyKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, unsigned _size, bool isSigned, bool isLittleEndian);
+ECLRTL_API IKeySegmentMonitor *createSingleKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned _offset, unsigned _size, const void * value);
+ECLRTL_API IKeySegmentMonitor *createSingleBigSignedKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned offset, unsigned size, const void * value);
+ECLRTL_API IKeySegmentMonitor *createSingleLittleSignedKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned offset, unsigned size, const void * value);
+ECLRTL_API IKeySegmentMonitor *createSingleLittleKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned offset, unsigned size, const void * value);
 
-//takes over ownership of both arguments
-ECLRTL_API IKeySegmentMonitor *createTranslatedKeySegmentMonitor(IKeySegmentMonitor * base, unsigned offset, IKeySegmentFormatTranslator * translator);
+ECLRTL_API IOverrideableKeySegmentMonitor *createOverrideableKeySegmentMonitor(IKeySegmentMonitor *base);
 
 ECLRTL_API IKeySegmentMonitor *deserializeKeySegmentMonitor(MemoryBuffer &mb);
 ECLRTL_API void deserializeSet(IStringSet & set, size32_t minRecordSize, const RtlTypeInfo * fieldType, const char * filter);

system/jhtree/jhtree.cpp

@@ -110,7 +110,7 @@ size32_t SegMonitorList::getSize() const
 void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname)
 {
     size32_t segSize = getSize();
-    if (segSize > keyedSize)
+    if (segSize != keyedSize)
     {
         StringBuffer err;
         err.appendf("Key size mismatch on key %s - key size is %u, expected %u", keyname, keyedSize, getSize());
@@ -118,8 +118,6 @@ void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname)
         EXCLOG(e, err.str());
         throw e;
     }
-    else if (segSize < keyedSize)
-        segMonitors.append(*createWildKeySegmentMonitor(segSize, keyedSize-segSize));
 }
 
 void SegMonitorList::setLow(unsigned segno, void *keyBuffer) const
@@ -215,44 +213,18 @@ unsigned SegMonitorList::lastFullSeg() const
     return ret;
 }
 
-void SegMonitorList::finish()
+void SegMonitorList::finish(unsigned keyedSize) // MORE - probably need to know the record layout here, if wilds are to be added properly.
 {
     if (modified)
     {
-        unsigned len = segMonitors.length();
-        unsigned curOffset = 0;
-        unsigned curSize = 0;
-        if (len)
+        size32_t segSize = getSize();
+        if (segSize < mergeBarrier)
         {
-            IKeySegmentMonitor *current = &segMonitors.item(0);
-            curOffset = current->getOffset();
-            curSize = current->getSize();
-            for (unsigned seg = 1; seg < len; seg++)
-            {
-                IKeySegmentMonitor *next = &segMonitors.item(seg);
-                unsigned nextOffset = next->getOffset();
-                unsigned nextSize = next->getSize();
-                assertex(curOffset + curSize == nextOffset) ;
-                IKeySegmentMonitor *merged = NULL;
-                if (nextOffset != mergeBarrier)
-                    merged = current->merge(next);
-                if (merged)
-                {
-                    current = merged;
-                    segMonitors.replace(*current, seg-1);
-                    segMonitors.remove(seg);
-                    curSize += nextSize;
-                    seg--;
-                    len--;
-                }
-                else
-                {
-                    current = next;
-                    curOffset = nextOffset;
-                    curSize = nextSize;
-                }
-            }
+            segMonitors.append(*createWildKeySegmentMonitor(0, segSize, mergeBarrier-segSize));  // MORE - field number is needed, but that requires layout info
+            segSize = mergeBarrier;
         }
+         if (segSize < keyedSize)
+            segMonitors.append(*createWildKeySegmentMonitor(0, segSize, keyedSize-segSize));  // MORE - field number is needed, but that requires layout info
         recalculateCache();
         modified = false;
     }
@@ -294,81 +266,40 @@ void SegMonitorList::serialize(MemoryBuffer &mb) const
 void SegMonitorList::append(IKeySegmentMonitor *segment)
 {
     modified = true;
+    unsigned fieldIdx = segment->getFieldIdx();
     unsigned offset = segment->getOffset();
     unsigned size = segment->getSize();
-    if (mergeBarrier)
+#if 0
+    // Eventually we will want something like this:
+    while (segMonitors.length() < fieldIdx)
     {
-        // check if we need to split
-        if (offset < mergeBarrier && offset+size > mergeBarrier)
-        {
-            IKeySegmentMonitor *split = segment->split(mergeBarrier - offset);
-            assertex(split);
-            append(split);   // note - recursive call
-            append(segment);
-            return;
-        }
+        unsigned idx = segMonitors.length();
+        IKeySegmentMonitor * before = createWildKeySegmentMonitor(idx, field(idx));
     }
+#else
+    // But as we don't have field info yet, and we don't care about it yet, this will do
+    unsigned lastOffset = 0;
     if (segMonitors.length())
     {
         unsigned lpos = segMonitors.length()-1;
         IKeySegmentMonitor &last = segMonitors.item(lpos);
-        if (last.getOffset() + last.getSize() == offset)
+        lastOffset = last.getOffset() + last.getSize();
+    }
+    assertex(lastOffset <= offset);
+    if (lastOffset < offset && needWild)
+    {
+        if (mergeBarrier && lastOffset < mergeBarrier && offset > mergeBarrier)
         {
-            segMonitors.append(*segment);
+            segMonitors.append(*createWildKeySegmentMonitor(0, lastOffset, mergeBarrier-lastOffset));
+            segMonitors.append(*createWildKeySegmentMonitor(0, mergeBarrier, offset-mergeBarrier));
         }
         else
         {
-            // we need to combine new segmonitor with existing segmonitors
-            assertex (offset + size <= last.getOffset() + last.getSize());   // no holes should ever be created...
-            if (!segment->isWild())  // X and wild is always X
-            {
-                // Find segmonitor that overlaps the new one, splitting if necessary
-                unsigned idx = 0;
-                while (segMonitors.isItem(idx))
-                {
-                    IKeySegmentMonitor &existing = segMonitors.item(idx);
-                    unsigned existingOffset = existing.getOffset();
-                    unsigned existingSize = existing.getSize();
-                    if (existingOffset <= offset && existingOffset+existingSize > offset)
-                    {
-                        // There is some overlap ...
-                        if (existingOffset < offset)
-                        {
-                            // split existing at start
-                            IKeySegmentMonitor *splitExisting = existing.split(offset-existingOffset);
-                            segMonitors.add(*splitExisting, idx);
-                            // now loop around to handle second half of existing
-                        }
-                        else if (size > existingSize)
-                        {
-                            // split new, merge first half into existing, and loop around to merge second half
-                            Owned<IKeySegmentMonitor> splitNew = segment->split(existingSize);
-                            IKeySegmentMonitor *combined = existing.combine(splitNew);
-                            segMonitors.replace(*combined, idx);
-                        }
-                        else if (size < existingSize)
-                        {
-                            // split existing, then merge new into first half
-                            Owned<IKeySegmentMonitor> split = existing.split(size); // leaves existing in the array as the tail
-                            IKeySegmentMonitor *combined = segment->combine(split);
-                            segMonitors.add(*combined, idx);
-                            break;
-                        }
-                        else // perfect overlap
-                        {
-                            IKeySegmentMonitor *combined = segment->combine(&existing);
-                            segMonitors.add(*combined, idx);
-                            break;
-                        }
-                    }
-                    idx++;
-                }
-            }
-            segment->Release();
+            segMonitors.append(*createWildKeySegmentMonitor(0, lastOffset, offset-lastOffset));
         }
     }
-    else
-        segMonitors.append(*segment);
+#endif
+    segMonitors.append(*segment);
 }
 
 bool SegMonitorList::matched(void *keyBuffer, unsigned &lastMatch) const
@@ -553,7 +484,7 @@ protected:
 public:
     IMPLEMENT_IINTERFACE;
 
-    CKeyLevelManager(IKeyIndex * _key, IContextLogger *_ctx)
+    CKeyLevelManager(IKeyIndex * _key, IContextLogger *_ctx) : segs(true)
     {
         ctx = _ctx;
         numsegs = 0;
@@ -653,8 +584,7 @@ public:
             {
                 started = true;
                 numsegs = segs.ordinality();
-                if (numsegs)
-                    segs.checkSize(keyedSize, keyName.get());
+                segs.checkSize(keyedSize, keyName.get());
             }
             if (!crappyHack)
             {
@@ -668,7 +598,7 @@ public:
 
     virtual void releaseSegmentMonitors()
     {
-        segs.segMonitors.kill();    
+        segs.reset();
         started = false;
         if(layoutTransSegCtx)
             layoutTransSegCtx->reset();
@@ -986,6 +916,7 @@ public:
 
     virtual void finishSegmentMonitors()
     {
+        segs.finish(keyedSize);
         if(transformSegs)
         {
             layoutTrans->createDiskSegmentMonitors(*layoutTransSegCtx, segs);
@@ -997,7 +928,6 @@ public:
                 throw MakeStringExceptionDirect(0, err.str());
             }
         }
-        segs.finish();
     }
 };
 
@@ -2425,7 +2355,6 @@ class CKeyMerger : public CKeyLevelManager
             }
             IKeySegmentMonitor *override = createOverrideableKeySegmentMonitor(LINK(&seg));
             segs.segMonitors.replace(*override, idx);
-
         }
         if (sortFromSeg == -1)
             assertex(!"Attempting to sort from offset that is not on a segment boundary"); // MORE - can use the information that we have earlier to make sure not merged
@@ -2958,7 +2887,11 @@ public:
     {
         CKeyLevelManager::finishSegmentMonitors();
         if (sortFieldOffset)
+        {
+            if (keyedSize)
+                segs.checkSize(keyedSize, "[merger]"); // Ensures trailing KSM is setup
             replicateForTrailingSort();
+        }
     }
 };
 
@@ -3054,14 +2987,14 @@ class IKeyManagerTest : public CppUnit::TestFixture
             Owned<IStringSet> sset1 = createStringSet(7);
             sset1->addRange("0000003", "0000003");
             sset1->addRange("0000005", "0000006");
-            tlk1->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
+            tlk1->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
             Owned<IStringSet> sset2 = createStringSet(3);
             sset2->addRange("010", "010");
             sset2->addRange("030", "033");
             Owned<IStringSet> sset3 = createStringSet(3);
             sset3->addRange("999", "XXX");
             sset3->addRange("000", "002");
-            tlk1->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
+            tlk1->append(createKeySegmentMonitor(false, sset2.getLink(), 1, 7, 3));
             tlk1->finishSegmentMonitors();
 
             tlk1->reset();
@@ -3090,8 +3023,8 @@ class IKeyManagerTest : public CppUnit::TestFixture
             Owned <IKeyManager> tlk2 = createKeyMerger(NULL, 7, NULL);
             tlk2->setKey(keyset);
             tlk2->deserializeCursorPos(mb);
-            tlk2->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
-            tlk2->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
+            tlk2->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
+            tlk2->append(createKeySegmentMonitor(false, sset2.getLink(), 1, 7, 3));
             tlk2->finishSegmentMonitors();
             tlk2->reset(true);
             ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000003032", 10)==0);
@@ -3105,8 +3038,8 @@ class IKeyManagerTest : public CppUnit::TestFixture
 
             Owned <IKeyManager> tlk3 = createKeyMerger(NULL, 7, NULL);
             tlk3->setKey(keyset);
-            tlk3->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
-            tlk3->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
+            tlk3->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
+            tlk3->append(createKeySegmentMonitor(false, sset2.getLink(), 1, 7, 3));
             tlk3->finishSegmentMonitors();
             tlk3->reset(false);
             ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000003010", 10)==0);
@@ -3118,8 +3051,8 @@ class IKeyManagerTest : public CppUnit::TestFixture
 
             Owned <IKeyManager> tlk4 = createKeyMerger(NULL, 7, NULL);
             tlk4->setKey(keyset);
-            tlk4->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
-            tlk4->append(createKeySegmentMonitor(false, sset3.getLink(), 7, 3));
+            tlk4->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
+            tlk4->append(createKeySegmentMonitor(false, sset3.getLink(), 1, 7, 3));
             tlk4->finishSegmentMonitors();
             tlk4->reset(false);
             ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003000", 10)==0);
@@ -3244,18 +3177,18 @@ protected:
             Owned <IKeyManager> tlk1 = createLocalKeyManager(index1, NULL);
             Owned<IStringSet> sset1 = createStringSet(10);
             sset1->addRange("0000000001", "0000000100");
-            tlk1->append(createKeySegmentMonitor(false, sset1.getClear(), 0, 10));
+            tlk1->append(createKeySegmentMonitor(false, sset1.getClear(), 0, 0, 10));
             tlk1->finishSegmentMonitors();
             tlk1->reset();
 
             Owned <IKeyManager> tlk1a = createLocalKeyManager(index1, NULL);
             Owned<IStringSet> sset1a = createStringSet(8);
             sset1a->addRange("00000000", "00000001");
-            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 0, 8));
-            tlk1a->append(createKeySegmentMonitor(false, NULL, 8, 1));
+            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 0, 0, 8));
+            tlk1a->append(createKeySegmentMonitor(false, NULL, 1, 8, 1));
             sset1a.setown(createStringSet(1));
             sset1a->addRange("0", "1");
-            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 9, 1));
+            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 2, 9, 1));
             tlk1a->finishSegmentMonitors();
             tlk1a->reset();
 
@@ -3285,7 +3218,7 @@ protected:
             Owned<IStringSet> sset2 = createStringSet(10);
             sset2->addRange("0000000001", "0000000100");
             ASSERT(sset2->numValues() == 65536);
-            tlk2->append(createKeySegmentMonitor(false, sset2.getClear(), 0, 10));
+            tlk2->append(createKeySegmentMonitor(false, sset2.getClear(), 0, 0, 10));
             tlk2->finishSegmentMonitors();
             tlk2->reset();
 
@@ -3299,7 +3232,7 @@ protected:
                 tlk3.setown(createKeyMerger(NULL, 0, NULL));
                 tlk3->setKey(both);
                 sset3->addRange("0000000001", "0000000100");
-                tlk3->append(createKeySegmentMonitor(false, sset3.getClear(), 0, 10));
+                tlk3->append(createKeySegmentMonitor(false, sset3.getClear(), 0, 0, 10));
                 tlk3->finishSegmentMonitors();
                 tlk3->reset();
             }
@@ -3308,7 +3241,7 @@ protected:
             Owned<IStringSet> sset2a = createStringSet(10);
             sset2a->addRange("0000000048", "0000000048");
             ASSERT(sset2a->numValues() == 1);
-            tlk2a->append(createKeySegmentMonitor(false, sset2a.getClear(), 0, 10));
+            tlk2a->append(createKeySegmentMonitor(false, sset2a.getClear(), 0, 0, 10));
             tlk2a->finishSegmentMonitors();
             tlk2a->reset();
 
@@ -3316,14 +3249,14 @@ protected:
             Owned<IStringSet> sset2b = createStringSet(10);
             sset2b->addRange("0000000047", "0000000049");
             ASSERT(sset2b->numValues() == 3);
-            tlk2b->append(createKeySegmentMonitor(false, sset2b.getClear(), 0, 10));
+            tlk2b->append(createKeySegmentMonitor(false, sset2b.getClear(), 0, 0, 10));
             tlk2b->finishSegmentMonitors();
             tlk2b->reset();
 
             Owned <IKeyManager> tlk2c = createLocalKeyManager(index2, NULL);
             Owned<IStringSet> sset2c = createStringSet(10);
             sset2c->addRange("0000000047", "0000000047");
-            tlk2c->append(createKeySegmentMonitor(false, sset2c.getClear(), 0, 10));
+            tlk2c->append(createKeySegmentMonitor(false, sset2c.getClear(), 0, 0, 10));
             tlk2c->finishSegmentMonitors();
             tlk2c->reset();
 

system/jhtree/jhtree.hpp

@@ -161,9 +161,10 @@ class jhtree_decl SegMonitorList : implements IInterface, implements IIndexReadC
     unsigned cachedLRS;
     unsigned mergeBarrier;
     bool modified;
+    bool needWild;
 public:
     IMPLEMENT_IINTERFACE;
-    inline SegMonitorList() { reset(); }
+    inline SegMonitorList(bool _needWild) : needWild(_needWild) { reset(); }
     IArrayOf<IKeySegmentMonitor> segMonitors;
 
     void reset();
@@ -180,7 +181,7 @@ public:
 
     void checkSize(size32_t keyedSize, char const * keyname);
     void recalculateCache();
-    void finish();
+    void finish(size32_t keyedSize);
     void deserialize(MemoryBuffer &mb);
     void serialize(MemoryBuffer &mb) const;
 

testing/regress/ecl/diskread.ecl

@@ -104,7 +104,7 @@ output(DG_FlatFile_pl0(KEYED(DG_firstname='CLAIRE')));
 output(DG_FlatFileEvens_pl0(KEYED(DG_firstname='CLAIRE')));
 //output(DG_CSVFile_pl0(KEYED(DG_firstname='CLAIRE')));
 //output(DG_XMLFile_pl0(KEYED(DG_firstname='CLAIRE')));
-output(DG_VarFile_pl0(KEYED(DG_firstname='CLAIRE')));
+//output(DG_VarFile_pl0(KEYED(DG_firstname='CLAIRE')));
 
 /*
 // now repeat with preload(n)....
@@ -145,7 +145,7 @@ output(DG_FlatFile_pl2(KEYED(DG_firstname='CLAIRE')));
 output(DG_FlatFileEvens_pl2(KEYED(DG_firstname='CLAIRE')));
 //output(DG_CSVFile_pl2(KEYED(DG_firstname='CLAIRE')));
 //output(DG_XMLFile_pl2(KEYED(DG_firstname='CLAIRE')));
-output(DG_VarFile_pl2(KEYED(DG_firstname='CLAIRE')));
+//output(DG_VarFile_pl2(KEYED(DG_firstname='CLAIRE')));
 
 */
 

testing/regress/ecl/key/diskread.xml

@@ -106,21 +106,3 @@
  <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>350</filepos></Row>
  <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>375</filepos></Row>
 </Dataset>
-<Dataset name='Result 12'>
- <Row><emptyfield></emptyfield><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><__filepos>624</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><extrafield>BACL                </extrafield><__filepos>653</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><__filepos>702</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><extrafield>BAYLCLAI            </extrafield><__filepos>731</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><__filepos>780</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><extrafield>DOCL                </extrafield><__filepos>809</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><__filepos>858</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><extrafield>DOLSCLAI            </extrafield><__filepos>887</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><__filepos>936</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><extrafield>BICL                </extrafield><__filepos>965</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><__filepos>1014</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><extrafield>BILLCLAI            </extrafield><__filepos>1043</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><__filepos>1092</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><extrafield>SMCL                </extrafield><__filepos>1121</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><__filepos>1170</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><extrafield>SMITCLAI            </extrafield><__filepos>1199</__filepos></Row>
-</Dataset>