Merge pull request #10660 from richardkchapman/remove-translated-segmons

HPCC-18755 Remove translated segMonitor support

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/fileview2/fvidxsource.cpp

@@ -432,7 +432,7 @@ void IndexDataSource::applyFilter()
     {
         IStringSet & cur = values.item(i);
         bool extend = true; // almost certainly better
-        manager->append(createKeySegmentMonitor(extend, LINK(&cur), keyedOffsets.item(i), fieldSize(i)));
+        manager->append(createKeySegmentMonitor(extend, LINK(&cur), i, keyedOffsets.item(i), fieldSize(i)));
     }
     manager->finishSegmentMonitors();
 

common/remote/sockfile.cpp

@@ -2996,7 +2996,7 @@ class CRemoteKeyManager : public CSimpleInterfaceOf<IKeyManager>
         return count;
     }
 public:
-    CRemoteKeyManager(const char *_filename, unsigned _crc, IDelayedFile *_delayedFile) : filename(_filename), crc(_crc), delayedFile(_delayedFile)
+    CRemoteKeyManager(const char *_filename, unsigned _crc, IDelayedFile *_delayedFile) : segs(true), filename(_filename), crc(_crc), delayedFile(_delayedFile)
     {
     }
     ~CRemoteKeyManager()
@@ -4826,7 +4826,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
         mb.read(handle).read(keyName).read(keySize);
         if (segmentMonitors)
         {
-            SegMonitorList segs;
+            SegMonitorList segs(true);
             segs.deserialize(mb);
             return prepKey(handle, keyName, &segs);
         }

common/thorhelper/layouttrans.cpp

@@ -508,7 +508,7 @@ void CRecordLayoutTranslator::createDiskSegmentMonitors(SegmentMonitorContext co
             //fall through
 
         case FieldMapping::None:
-            monitor.setown(createWildKeySegmentMonitor(diskOffset, size));
+            monitor.setown(createWildKeySegmentMonitor(diskFieldNum, diskOffset, size));
             break;
 
         case FieldMapping::ChildDataset:
@@ -542,15 +542,10 @@ size32_t CRecordLayoutTranslator::transformRow(RowTransformContext * ctx, byte c
 void ExpandedSegmentMonitorList::append(IKeySegmentMonitor * monitor)
 {
     if(owner->failure) return;
-    while(monitor->getSize() > owner->activityKeySizes[monitors.ordinality()])
+    if(monitor->getSize() > owner->activityKeySizes[monitors.ordinality()])
     {
-        Owned<IKeySegmentMonitor> split = monitor->split(owner->activityKeySizes[monitors.ordinality()]);
-        if(!split)
-        {
-            owner->failure.setown(makeFailure(IRecordLayoutTranslator::Failure::UnsupportedFilter)->append("Unsupported filter (segment monitor) type (was larger than keyed field and unsplittable)"));
-            return;
-        }
-        monitors.append(*split.getLink());
+        owner->failure.setown(makeFailure(IRecordLayoutTranslator::Failure::UnsupportedFilter)->append("Unsupported filter (segment monitor) type (was larger than keyed field)"));
+        return;
     }
     if(monitor->getSize() < owner->activityKeySizes[monitors.ordinality()])
     {

ecl/hqlcpp/hqlcatom.cpp

@@ -92,7 +92,6 @@ IAtom * _spill_Atom;
 IAtom * _spillReason_Atom;
 IAtom * _steppedMeta_Atom;
 IAtom * subgraphAtom;
-IAtom * _translated_Atom;
 IAtom * userFunctionAtom;
 IAtom * utf8Atom;
 IAtom * wrapperAtom;
@@ -1501,6 +1500,5 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKESYSATOM(spill);
     MAKESYSATOM(spillReason);
     MAKESYSATOM(steppedMeta);
-    MAKESYSATOM(translated);
     return true;
 }

ecl/hqlcpp/hqlcatom.hpp

@@ -90,7 +90,6 @@ extern IAtom * _spill_Atom;
 extern IAtom * _spillReason_Atom;
 extern IAtom * _steppedMeta_Atom;
 extern IAtom * subgraphAtom;
-extern IAtom * _translated_Atom;
 extern IAtom * userFunctionAtom;
 extern IAtom * utf8Atom;
 extern IAtom * wrapperAtom;

ecl/hqlcpp/hqlsource.cpp

@@ -3714,10 +3714,9 @@ IHqlExpression * KeyConditionInfo::createConjunction()
     return result.getClear();
 }
 
-MonitorExtractor::MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _allowTranslatedConds) : translator(_translator) 
+MonitorExtractor::MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _isDiskRead) : translator(_translator)
 { 
     tableExpr = _tableExpr;
-    allowTranslatedConds = _allowTranslatedConds;
 
     if (_numKeyableFields <= 0)
     {
@@ -3732,7 +3731,7 @@ MonitorExtractor::MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator
     else
         numKeyableFields = (unsigned)_numKeyableFields;
 
-    onlyHozedCompares = !allowTranslatedConds;
+    onlyHozedCompares = !_isDiskRead;
 
     expandKeyableFields();
     cleanlyKeyedExplicitly = false;
@@ -3956,7 +3955,7 @@ bool MonitorExtractor::createGroupingMonitor(BuildCtx ctx, const char * listName
                 if (expr == &cur)
                 {
                     //MORE: Check the type of the field is legal.
-                    ctx.addQuotedF("%s->append(createWildKeySegmentMonitor(%u, %u));", listName, offset, curSize);
+                    ctx.addQuotedF("%s->append(createWildKeySegmentMonitor(%u, %u, %u));", listName, 0, offset, curSize);
                     maxOffset = offset+curSize;
                     return true;
                 }
@@ -4109,7 +4108,7 @@ void MonitorExtractor::buildKeySegmentInExpr(BuildMonitorState & buildState, Key
             OwnedHqlExpr test = createBoolExpr(no_eq, LINK(lhs), LINK(curValue));
             OwnedHqlExpr promoted = getExplicitlyPromotedCompare(test);
             OwnedHqlExpr compare, normalized;
-            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector, selectorInfo.isComputed);
+            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector);
             if (compare)
                 translator.buildFilter(subctx, compare);
 
@@ -4146,7 +4145,7 @@ void MonitorExtractor::buildKeySegmentInExpr(BuildMonitorState & buildState, Key
             OwnedHqlExpr test = createBoolExpr(no_eq, LINK(lhs), LINK(cur));
             OwnedHqlExpr promoted = getExplicitlyPromotedCompare(test);
             OwnedHqlExpr compare, normalized;
-            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector, selectorInfo.isComputed);
+            extractCompareInformation(subctx, promoted, compare, normalized, expandedSelector);
             if (compare)
                 translator.buildFilter(subctx, compare);
 
@@ -4169,36 +4168,31 @@ static IHqlExpression * createCompareRecast(node_operator op, IHqlExpression * v
 }
 
 
-void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isComputed)
+void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector)
 {
-    extractCompareInformation(ctx, expr->queryChild(0), expr->queryChild(1), compare, normalized, expandedSelector, isComputed);
+    extractCompareInformation(ctx, expr->queryChild(0), expr->queryChild(1), compare, normalized, expandedSelector);
 }
 
 
-void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isComputed)
+void MonitorExtractor::extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector)
 {
     LinkedHqlExpr compareValue = value->queryBody();
     OwnedHqlExpr recastValue;
-    if (isComputed)
-        normalized.setown(ensureExprType(compareValue, expandedSelector->queryType()));
-    else
+    if ((lhs->getOperator() != no_select) || (lhs->queryType() != compareValue->queryType()))
     {
-        if ((lhs->getOperator() != no_select) || (lhs->queryType() != compareValue->queryType()))
+        OwnedHqlExpr temp  = castToFieldAndBack(lhs, compareValue);
+        if (temp != compareValue)
         {
-            OwnedHqlExpr temp  = castToFieldAndBack(lhs, compareValue);
-            if (temp != compareValue)
-            {
-                //Force into a temporary variable since it will be used more than once, and reapply the field casting/
-                compareValue.setown(translator.buildSimplifyExpr(ctx, compareValue));
-                //cast to promoted type because sometimes evaluating can convert string to string<n>
-                Owned<ITypeInfo> promotedType = getPromotedECLType(lhs->queryType(), compareValue->queryType());
-                compareValue.setown(ensureExprType(compareValue, promotedType));
-                recastValue.setown(castToFieldAndBack(lhs, compareValue));
-            }
+            //Force into a temporary variable since it will be used more than once, and reapply the field casting/
+            compareValue.setown(translator.buildSimplifyExpr(ctx, compareValue));
+            //cast to promoted type because sometimes evaluating can convert string to string<n>
+            Owned<ITypeInfo> promotedType = getPromotedECLType(lhs->queryType(), compareValue->queryType());
+            compareValue.setown(ensureExprType(compareValue, promotedType));
+            recastValue.setown(castToFieldAndBack(lhs, compareValue));
         }
-
-        normalized.setown(invertTransforms(lhs, compareValue));
     }
+
+    normalized.setown(invertTransforms(lhs, compareValue));
     normalized.setown(foldHqlExpression(normalized));
 
     if (recastValue && recastValue != compareValue)
@@ -4232,7 +4226,7 @@ void MonitorExtractor::buildKeySegmentCompareExpr(BuildMonitorState & buildState
     OwnedHqlExpr compare;
     OwnedHqlExpr normalized;
     BuildCtx subctx(ctx);
-    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector, selectorInfo.isComputed);
+    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector);
     OwnedHqlExpr address = getMonitorValueAddress(subctx, normalized);
 
     HqlExprArray args;
@@ -4445,8 +4439,6 @@ void MonitorExtractor::buildKeySegmentExpr(BuildMonitorState & buildState, KeySe
                         OwnedHqlExpr targetVar = createVariable(targetSet, makeVoidType());
                         callAddAll(subctx, targetVar);
                     }
-                    else
-                        buildWildKeySegment(buildState, subctx, selectorInfo);
                     subctx.selectElse(ifStmt);
                 }
             }
@@ -4489,22 +4481,14 @@ void MonitorExtractor::buildKeySegmentExpr(BuildMonitorState & buildState, KeySe
 
     if (targetSet && !requiredSet)
     {
-        unsigned offset = (selectorInfo.isComputed || selectorInfo.mapOffset) ? 0 : selectorInfo.offset;
-        createMonitorText.appendf("createKeySegmentMonitor(%s, %s.getClear(), %u, %u)", 
-                                  boolToText(selectorInfo.keyedKind != KeyedYes), targetSet, offset, selectorInfo.size);
+        createMonitorText.appendf("createKeySegmentMonitor(%s, %s.getClear(), %u, %u, %u)",
+                                  boolToText(selectorInfo.keyedKind != KeyedYes), targetSet, selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size);
 
         buildState.popSetName();
     }
 
     if (createMonitorText.length())
-    {
-        if (selectorInfo.expandNeeded || selectorInfo.isComputed)
-            generateFormatWrapping(createMonitorText, selectorInfo.selector, selectorInfo.expandedSelector, buildState.curOffset);
-        else if (selectorInfo.mapOffset)
-            generateOffsetWrapping(createMonitorText, selectorInfo.selector);
-
         appendCtx->addQuotedF("%s->append(%s);", buildState.listName, createMonitorText.str());
-    }
 }
 
 
@@ -4540,85 +4524,13 @@ IHqlExpression * MonitorExtractor::getMonitorValueAddress(BuildCtx & ctx, IHqlEx
     return getPointer(bound.expr);
 }
 
-
-/*
-interface IKeySegmentOffsetTranslator : public IInterface
-{
-    virtual const void * getSegmentBase(const void * row) const = 0;
-};
-
-interface IKeySegmentFormatTranslator : public IInterface
-{
-    virtual void extractField(void * target, const void * row) const = 0;
-};
-*/
-
-void MonitorExtractor::generateOffsetWrapping(StringBuffer & createMonitorText, IHqlExpression * selector)
-{
-    unsigned curFieldIdx = getFieldNumber(tableExpr->queryNormalizedSelector(), selector);
-    StringBuffer s;
-    s.clear().append("createNewVarOffsetKeySegmentMonitor(").append(createMonitorText).append(",").append(curFieldIdx).append(")");
-    createMonitorText.swapWith(s);
-}
-
-void MonitorExtractor::generateFormatWrapping(StringBuffer & createMonitorText, IHqlExpression * selector, IHqlExpression * expandedSelector, unsigned curOffset)
-{
-    BuildCtx declarectx(*translator.code, declareAtom);
-    StringBuffer s, suffix, instanceName, className, factoryName;
-    unique_id_t id = translator.getUniqueId();
-    appendUniqueId(className.append("c"), id);
-    appendUniqueId(instanceName.append("i"), id);
-    appendUniqueId(factoryName.append("f"), id);
-
-    declarectx.setNextPriority(SegMonitorPrio);
-    BuildCtx classctx(declarectx);
-    //MORE: Use a base class for implementing this to save Link()/Release()
-    s.clear().append("struct ").append(className).append(" : public RtlCInterface, public IKeySegmentFormatTranslator");
-    suffix.append(instanceName).append(";");
-    classctx.addQuotedCompound(s, suffix);
-    classctx.addQuotedLiteral("virtual void Link() const override { RtlCInterface::Link(); }");
-    classctx.addQuotedLiteral("virtual bool Release() const override { return RtlCInterface::Release(); }");
-    classctx.addQuoted(s.clear().append("virtual const char * queryFactoryName() const override { return \"").append(factoryName).append("\"; }"));
-    classctx.addQuoted(s.clear().append("virtual unsigned queryHashCode() const override { return ").append(getExpressionCRC(selector)).append("; }"));
-
-    {
-        MemberFunction func(translator, classctx, "virtual void extractField(void * _target, const void * _row) const override");
-        classctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
-        func.ctx.addQuotedLiteral("const byte * row = (const byte *)_row;");
-        func.ctx.addQuotedLiteral("byte * target = (byte *)_target;");
-
-        OwnedHqlExpr castValue = ensureExprType(selector, expandedSelector->queryType());
-        LinkedHqlExpr targetField = expandedSelector->queryChild(1);
-        OwnedHqlExpr simpleRecord = createRecord(targetField);
-        OwnedHqlExpr targetDataset = createDataset(no_anon, LINK(simpleRecord));
-        OwnedHqlExpr target = createSelectExpr(LINK(targetDataset), LINK(targetField));
-
-        translator.bindTableCursor(func.ctx, tableExpr, "row");
-        translator.bindTableCursor(func.ctx, targetDataset, "target");
-        translator.buildAssign(func.ctx, target, castValue);
-    }
-
-    declarectx.setNextPriority(SegMonitorPrio);
-    declarectx.addQuoted(s.clear().append("IKeySegmentFormatTranslator * ").append(factoryName).append("() { return new ").append(className).append("; }"));
-    if (translator.spanMultipleCppFiles())
-    {
-        s.clear().append("extern IKeySegmentFormatTranslator * ").append(factoryName).append("();");
-        BuildCtx protoctx(*translator.code, mainprototypesAtom);
-        protoctx.addQuoted(s);
-    }
-
-    //Now generate the key segment monitor...
-    s.clear().append("createTranslatedKeySegmentMonitor(").append(createMonitorText).append(",").append(curOffset).append(",").append(factoryName).append("())");
-    createMonitorText.swapWith(s);
-}
-
 bool MonitorExtractor::buildSingleKeyMonitor(StringBuffer & createMonitorText, KeySelectorInfo & selectorInfo, BuildCtx & ctx, IHqlExpression & thisKey)
 {
     BuildCtx subctx(ctx);
     OwnedHqlExpr compare, normalized;
 
     StringBuffer funcName;
-    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector, selectorInfo.isComputed);
+    extractCompareInformation(subctx, &thisKey, compare, normalized, selectorInfo.expandedSelector);
     if (compare)
         return false;
 
@@ -4649,10 +4561,9 @@ bool MonitorExtractor::buildSingleKeyMonitor(StringBuffer & createMonitorText, K
     StringBuffer addrText;
     translator.generateExprCpp(addrText, address);
 
-    unsigned offset = (selectorInfo.isComputed || selectorInfo.mapOffset) ? 0 : selectorInfo.offset;
     createMonitorText.append(funcName)
-                     .appendf("(%s, %u, %u, %s)", 
-                              boolToText(selectorInfo.keyedKind != KeyedYes), offset, selectorInfo.size, addrText.str());
+                     .appendf("(%s, %u, %u, %u, %s)",
+                              boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size, addrText.str());
     return true;
 }
 
@@ -4676,20 +4587,7 @@ KeyedKind getKeyedKind(HqlCppTranslator & translator, KeyConditionArray & matche
 void MonitorExtractor::buildEmptyKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo)
 {
     StringBuffer s;
-    ctx.addQuoted(s.appendf("%s->append(createEmptyKeySegmentMonitor(%s, %u, %u));", buildState.listName, boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.offset, selectorInfo.size));
-}
-
-
-void MonitorExtractor::buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned offset, unsigned size)
-{
-    StringBuffer s;
-    ctx.addQuoted(s.appendf("%s->append(createWildKeySegmentMonitor(%u, %u));", buildState.listName, offset, size));
-}
-
-
-void MonitorExtractor::buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo)
-{
-    buildWildKeySegment(buildState, ctx, selectorInfo.offset, selectorInfo.size);
+    ctx.addQuoted(s.appendf("%s->append(createEmptyKeySegmentMonitor(%s, %u, %u, %u));", buildState.listName, boolToText(selectorInfo.keyedKind != KeyedYes), selectorInfo.fieldIdx, selectorInfo.offset, selectorInfo.size));
 }
 
 
@@ -4744,7 +4642,9 @@ void MonitorExtractor::buildKeySegment(BuildMonitorState & buildState, BuildCtx
 
     StringBuffer s;
     KeyedKind keyedKind = getKeyedKind(translator, matches);
-    KeySelectorInfo selectorInfo(keyedKind, selector, expandedSelector, buildState.curOffset, curSize, (whichField >= firstOffsetField), false);
+    if (whichField >= firstOffsetField)
+        translator.throwError1(HQLERR_KeyedNotKeyed, getExprECL(field, s).str());
+    KeySelectorInfo selectorInfo(keyedKind, selector, expandedSelector, buildState.curFieldIdx, buildState.curOffset, curSize);
 
     bool ignoreKeyedExtend = false;
     if ((keyedKind == KeyedExtend) && buildState.wildPending() && !ignoreUnkeyed)
@@ -4769,10 +4669,7 @@ void MonitorExtractor::buildKeySegment(BuildMonitorState & buildState, BuildCtx
     if (matches.ordinality() && !ignoreKeyedExtend)
     {
         if (buildState.wildPending() && !ignoreUnkeyed)
-        {
-            buildWildKeySegment(buildState, ctx, buildState.wildOffset, buildState.curOffset-buildState.wildOffset);
             buildState.clearWild();
-        }
 
         HqlExprArray args;
         ForEachItemIn(i, matches)
@@ -4794,32 +4691,15 @@ void MonitorExtractor::buildKeySegment(BuildMonitorState & buildState, BuildCtx
         }
 
         if (buildState.wildPending() && noMergeSelects.contains(*selector))
-        {
-            buildWildKeySegment(buildState, ctx, buildState.wildOffset, buildState.curOffset-buildState.wildOffset);
             buildState.clearWild();
-        }
 
         if (!buildState.wildPending())
             buildState.wildOffset = buildState.curOffset;
     }
     buildState.curOffset += selectorInfo.size;
+    buildState.curFieldIdx++;
 }
 
-void MonitorExtractor::buildArbitaryKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned curSize, IHqlExpression * condition)
-{
-    IHqlExpression * left = condition->queryChild(0);
-    node_operator op = condition->getOperator();
-
-    StringBuffer createMonitorText;
-    OwnedHqlExpr field = createField(unknownId, getExpandedFieldType(left->queryType(), NULL), NULL);
-    OwnedHqlExpr pseudoSelector = createSelectExpr(getActiveTableSelector(), LINK(field));
-
-    KeySelectorInfo selectorInfo(KeyedExtend, left, pseudoSelector, buildState.curOffset, curSize, false, true);
-    BuildCtx subctx(ctx);
-    buildKeySegmentExpr(buildState, selectorInfo, subctx, NULL, *condition, MonitorFilterSkipAll);
-}
-
-
 void MonitorExtractor::spotSegmentCSE(BuildCtx & ctx)
 {
     //This could make things much better, but needs some thought
@@ -4912,25 +4792,6 @@ void MonitorExtractor::buildSegments(BuildCtx & ctx, const char * listName, bool
         }
     }
 
-    if (buildState.wildPending() && !ignoreUnkeyed)
-        buildWildKeySegment(buildState, ctx, buildState.wildOffset, buildState.curOffset-buildState.wildOffset);
-
-    //These really don't work very sensibly - we would need an offset that was constant, possibly based on the crc of the expression.
-    //I suspect they need a complete rethink.
-    ForEachItemIn(i, keyed.conditions)
-    {
-        KeyCondition & cur = keyed.conditions.item(i);
-        if (cur.selector->isAttribute() && cur.selector->queryName() == _translated_Atom)
-        {
-            BuildCtx subctx(ctx);
-            IHqlExpression * curExpr = cur.expr;
-            unsigned curSize = curExpr->queryChild(0)->queryType()->getSize();
-            buildArbitaryKeySegment(buildState, subctx, curSize, curExpr);
-            buildState.curOffset += curSize;
-            cur.generated = true;
-        }
-    }
-
     //check that all keyed entries have been matched
     ForEachItemIn(cond, keyed.conditions)
     {
@@ -4941,13 +4802,6 @@ void MonitorExtractor::buildSegments(BuildCtx & ctx, const char * listName, bool
 }
 
 
-static UniqueSequenceCounter translatedSequence;
-KeyCondition * MonitorExtractor::createTranslatedCondition(IHqlExpression * cond, KeyedKind keyedKind)
-{
-    OwnedHqlExpr seq = createSequence(no_attr, makeNullType(), _translated_Atom, translatedSequence.next());
-    return new KeyCondition(seq, cond, keyedKind);
-}
-
 bool MonitorExtractor::isKeySelect(IHqlExpression * select)
 {
     return (keyableSelects.find(*select) != NotFound);
@@ -5663,26 +5517,6 @@ bool MonitorExtractor::extractSimpleCompareFilter(KeyConditionInfo & matches, IH
         }
     }
 
-    if (!result && allowTranslatedConds)
-    {
-        duplicate = false;
-        ITypeInfo * type = l->queryType();
-        if (isKeyableType(type))
-        {
-            bool leftHasSelects = containsTableSelects(l);
-            bool rightHasSelects = containsTableSelects(r);
-            if (leftHasSelects && !rightHasSelects)
-            {
-                result.setown(createTranslatedCondition(expr, keyedKind));
-            }
-            else if (!leftHasSelects && rightHasSelects && (op != no_in) && (op != no_notin))
-            {
-                OwnedHqlExpr newFilter = createValue(getReverseOp(op), expr->getType(), LINK(r), LINK(l));
-                result.setown(createTranslatedCondition(newFilter, keyedKind));
-            }
-        }
-    }
-
     bool extracted = (result != NULL);
     if (extracted)
     {

ecl/hqlcpp/hqlsource.ipp

@@ -80,6 +80,7 @@ struct BuildMonitorState
     BuildMonitorState(BuildCtx & _funcctx, const char * _listName) : funcctx(_funcctx) 
     { 
         listName = _listName; 
+        curFieldIdx = 0;
         curOffset = 0;
         wildOffset = (unsigned) -1;
         numActiveSets = 0;
@@ -105,6 +106,7 @@ struct BuildMonitorState
     bool doneImplicitWarning;
     bool warnedAllConditionsWild;
     bool wildWasKeyed;
+    unsigned curFieldIdx;
     unsigned curOffset;
     unsigned wildOffset;
 };
@@ -114,32 +116,28 @@ enum MonitorFilterKind { NoMonitorFilter, MonitorFilterSkipEmpty, MonitorFilterS
 struct KeySelectorInfo
 {
 public:
-    KeySelectorInfo(KeyedKind _keyedKind, IHqlExpression * _selector, IHqlExpression * _expandedSelector, size32_t _offset, size32_t _size, bool _mapOffset, bool _isComputed)
+    KeySelectorInfo(KeyedKind _keyedKind, IHqlExpression * _selector, IHqlExpression * _expandedSelector, unsigned _fieldIdx, size32_t _offset, size32_t _size)
     {
         keyedKind = _keyedKind; 
         selector = _selector; 
-        expandedSelector = _expandedSelector; 
+        expandedSelector = _expandedSelector;
+        fieldIdx = _fieldIdx;
         offset = _offset; 
         size = _size;
-        expandNeeded = (selector->queryType() != expandedSelector->queryType());
-        mapOffset = _mapOffset;
-        isComputed = _isComputed;
     }
 
     IHqlExpression * selector;
     IHqlExpression * expandedSelector;
+    unsigned fieldIdx;
     size32_t offset;
     size32_t size;
     KeyedKind keyedKind;
-    bool expandNeeded;
-    bool mapOffset;
-    bool isComputed;
 };
 
 class MonitorExtractor
 {
 public:
-    MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _allowTranslatedConds);
+    MonitorExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool isDiskRead);
 
     void appendFilter(IHqlExpression * expr)                { keyed.appendPostFilter(expr); }
     void buildSegments(BuildCtx & ctx, const char * listName, bool _ignoreUnkeyed);
@@ -164,10 +162,7 @@ public:
     bool createGroupingMonitor(BuildCtx ctx, const char * listName, IHqlExpression * select, unsigned & maxOffset);
 
 protected:
-    void buildArbitaryKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned curSize, IHqlExpression * condition);
     void buildEmptyKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo);
-    void buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, KeySelectorInfo & selectorInfo);
-    void buildWildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned offset, unsigned size);
     void buildKeySegment(BuildMonitorState & buildState, BuildCtx & ctx, unsigned whichField, unsigned curSize);
     void buildKeySegmentExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * target, IHqlExpression & thisKey, MonitorFilterKind filterKind);
     void buildKeySegmentCompareExpr(BuildMonitorState & buildState, KeySelectorInfo & selectorInfo, BuildCtx & ctx, const char * requiredSet, IHqlExpression & thisKey);
@@ -187,8 +182,6 @@ protected:
     void expandKeyableFields();
     void expandSelects(IHqlExpression * expr, IHqlSimpleScope * expandedScope, IHqlExpression * keySelector, IHqlExpression * expandedSelector);;
     bool extractOrFilter(KeyConditionInfo & matches, IHqlExpression * filter, KeyedKind keyedKind);
-    void generateFormatWrapping(StringBuffer & createMonitorText, IHqlExpression * selector, IHqlExpression * expandedSelector, unsigned curOffset);
-    void generateOffsetWrapping(StringBuffer & createMonitorText, IHqlExpression * selector);
     IHqlExpression * getMonitorValueAddress(BuildCtx & ctx, IHqlExpression * value);
     IHqlExpression * getRangeLimit(ITypeInfo * fieldType, IHqlExpression * lengthExpr, IHqlExpression * value, int whichBoundary);
     IHqlExpression * invertTransforms(IHqlExpression * left, IHqlExpression * right);
@@ -201,8 +194,8 @@ protected:
     bool okToKey(IHqlExpression * select, KeyedKind keyedKind);
     IHqlExpression * queryKeyableSelector(IHqlExpression * expr);
     IHqlExpression * querySimpleJoinValue(IHqlExpression * field);
-    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isTranslated);
-    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector, bool isTranslated);
+    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * expr, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
+    void extractCompareInformation(BuildCtx & ctx, IHqlExpression * lhs, IHqlExpression * value, SharedHqlExpr & compare, SharedHqlExpr & normalized, IHqlExpression * expandedSelector);
     IHqlExpression * unwindConjunction(HqlExprArray & matches, IHqlExpression * expr);
 
 protected:
@@ -238,7 +231,6 @@ protected:
     HqlExprCopyArray noMergeSelects;    // don't merge these fields (even for wildcards) because they are separate stepping fields.
     unsigned firstOffsetField;          // first field where the keyed offset is adjusted
     bool onlyHozedCompares;
-    bool allowTranslatedConds;
     bool ignoreUnkeyed;
     bool cleanlyKeyedExplicitly;
     bool keyedExplicitly;

roxie/ccd/ccdkey.cpp

@@ -269,11 +269,13 @@ public:
         ForEach(*fields)
         {
             IPropertyTree &field = fields->query();
+            unsigned fieldNum = field.getPropInt("@fieldNum", -1);
+            assertex(fieldNum != (unsigned) -1);
             unsigned offset = field.getPropInt("@offset", 0);
             unsigned size = field.getPropInt("@size", 0);
             bool isSigned = field.getPropBool("@isSigned", false);
             bool isLittleEndian = field.getPropBool("@isLittleEndian", false);
-            append(createDummyKeySegmentMonitor(offset, size, isSigned, isLittleEndian));
+            append(createDummyKeySegmentMonitor(fieldNum, offset, size, isSigned, isLittleEndian));
         }
     }
 
@@ -326,7 +328,7 @@ public:
 #endif
         ForEachItemIn(idx, segMonitors)
         {
-            int ret = segMonitors.item(idx).docompareraw(l, r);
+            int ret = segMonitors.item(idx).docompare(l, r);
             if (ret)
                 return ret;
         }
@@ -386,7 +388,7 @@ public:
         ForEachItemIn(idx, segMonitors)
         {
             IKeySegmentMonitor &item = segMonitors.item(idx);
-            ret.pad(indent+1).appendf("<Field offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", item.getOffset(), item.getSize(), item.isSigned(), item.isLittleEndian());
+            ret.pad(indent+1).appendf("<Field fieldNum='%d' offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", item.getFieldIdx(), item.getOffset(), item.getSize(), item.isSigned(), item.isLittleEndian());
         }
         ret.pad(indent).append("</FieldSet>\n");
         return ret;
@@ -979,8 +981,7 @@ class InMemoryIndexManager : implements IInMemoryIndexManager, public CInterface
             ForEachItemIn(idx, m)
             {
                 IKeySegmentMonitor &seg = m.item(idx);
-                xml.appendf("<Field offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", seg.getOffset(), seg.getSize(), seg.isSigned(), seg.isLittleEndian());
-//              xml.appendf("<Field offset='%d' size='%d' flags='%d'/>\n", seg.getOffset(), seg.getSize(), seg.getFlags());
+                xml.appendf("<Field fieldNum='%d' offset='%d' size='%d' isSigned='%d' isLittleEndian='%d'/>\n", seg.getFieldIdx(), seg.getOffset(), seg.getSize(), seg.isSigned(), seg.isLittleEndian());
             }
             xml.append("</FieldSet>\n");
         }
@@ -1307,11 +1308,12 @@ public:
             {
                 // MORE - we could share them, then comparison would be faster (pointer compare)
                 IKeySegmentMonitor &seg = perfect.item(idx);
+                unsigned fieldNum = seg.getFieldIdx();
                 unsigned offset = seg.getOffset();
                 unsigned size = seg.getSize();
                 bool isSigned = seg.isSigned();
                 bool isLittleEndian = seg.isLittleEndian();
-                newEntry->append(*createDummyKeySegmentMonitor(offset, size, isSigned, isLittleEndian));
+                newEntry->append(*createDummyKeySegmentMonitor(fieldNum, offset, size, isSigned, isLittleEndian));
             }
             tracked[a] = newEntry;
             hits[a] = noteHits ? noteHits : 1;
@@ -2054,14 +2056,14 @@ protected:
         di.load(testarray, sizeof(testarray), &x, 1);
         unsigned searchval = 1;
         InMemoryIndex order;
-        order.append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        order.append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         di.setBuilding();
         di.resort(order);
         InMemoryIndexManager indexes(false, "test1");
         indexes.append(*LINK(&di));
 
         Owned<IInMemoryIndexCursor> d = indexes.createCursor(dummy);
-        d->append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        d->append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         d->selectKey();
         d->reset();
         ASSERT(d->nextMatch()==&testarray[0]);
@@ -2070,7 +2072,7 @@ protected:
 
         d.setown(indexes.createCursor(dummy));
         searchval = 10;
-        d->append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        d->append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         d->selectKey();
         d->reset();
         ASSERT(d->nextMatch()==NULL);
@@ -2078,7 +2080,7 @@ protected:
         d.clear();
         di.deprecate();
 
-        Owned<IPropertyTree> keyInfo = createPTreeFromXMLString("<R><FieldSet><Field isLittleEndian='1' isSigned='0' offset='0' size='4'/><Field isLittleEndian='1' isSigned='1' offset='0' size='4'/></FieldSet></R>");
+        Owned<IPropertyTree> keyInfo = createPTreeFromXMLString("<R><FieldSet><Field isLittleEndian='1' isSigned='0' offset='0' fieldNum='0' size='4'/><Field isLittleEndian='1' isSigned='1' offset='0' fieldNum='0' size='4'/></FieldSet></R>");
         indexes.setKeyInfo(*keyInfo.get());
         Sleep(1000); // to give key time to build!
 
@@ -2088,7 +2090,7 @@ protected:
         set->addRange(&searchval, &searchval);
         searchval = 9;
         set->addRange(&searchval, &searchval);
-        d->append(createKeySegmentMonitor(false, set, 0, sizeof(unsigned)));
+        d->append(createKeySegmentMonitor(false, set, 0, 0, sizeof(unsigned)));
         d->selectKey();
         d->reset();
         ASSERT(*(int *) d->nextMatch()==2);
@@ -2108,19 +2110,19 @@ protected:
         const RtlFieldInfo * const fields [] = {&f1, nullptr};
         RtlRecord dummy(fields, true); // NOTE - not accurate but good enough for these tests
 
-        IKeySegmentMonitor *ksm = createSingleLittleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval);
+        IKeySegmentMonitor *ksm = createSingleLittleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval);
         ASSERT(ksm->isSigned()==false);
         ASSERT(ksm->isLittleEndian()==true);
         di.append(ksm);
-        di.append(createSingleKeySegmentMonitor(false, 4, sizeof(unsigned), &searchval));
-        di.append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        di.append(createSingleKeySegmentMonitor(false, 1, 4, sizeof(unsigned), &searchval));
+        di.append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         di.setBuilding();
         di.undeprecate();
 
         InMemoryIndex di2;
-        di2.append(createSingleKeySegmentMonitor(false, 12, sizeof(unsigned), &searchval));
-        di2.append(createSingleKeySegmentMonitor(false, 16, sizeof(unsigned), &searchval));
-        di2.append(createSingleKeySegmentMonitor(false, 0, sizeof(unsigned), &searchval));
+        di2.append(createSingleKeySegmentMonitor(false, 3, 12, sizeof(unsigned), &searchval));
+        di2.append(createSingleKeySegmentMonitor(false, 4, 16, sizeof(unsigned), &searchval));
+        di2.append(createSingleKeySegmentMonitor(false, 0, 0, sizeof(unsigned), &searchval));
         di2.setBuilding();
         di2.undeprecate();
         
@@ -2131,8 +2133,8 @@ protected:
         Owned<IInMemoryIndexCursor> dd = indexes.createCursor(dummy);
         InMemoryIndexCursor *d = QUERYINTERFACE(dd.get(), InMemoryIndexCursor);
         ASSERT(d != nullptr);
-        dd->append(createSingleKeySegmentMonitor(false, 4, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 1, 4, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval));
         ASSERT(d->postFilter.length()==2);
         ASSERT(d->segMonitors.length()==0);
         ASSERT(d->postFilter.item(0).getOffset()==4);
@@ -2146,8 +2148,8 @@ protected:
         dd.setown(indexes.createCursor(dummy));
         d = QUERYINTERFACE(dd.get(), InMemoryIndexCursor);
         ASSERT(d != nullptr);
-        dd->append(createSingleKeySegmentMonitor(false, 16, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 4, 16, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval));
         ASSERT(d->postFilter.length()==2);
         ASSERT(d->segMonitors.length()==0);
         ASSERT(d->postFilter.item(0).getOffset()==8);
@@ -2161,9 +2163,9 @@ protected:
         dd.setown(indexes.createCursor(dummy));
         d = QUERYINTERFACE(dd.get(), InMemoryIndexCursor);
         ASSERT(d != nullptr);
-        dd->append(createSingleKeySegmentMonitor(false, 12, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 16, sizeof(unsigned), &searchval));
-        dd->append(createSingleKeySegmentMonitor(false, 8, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 3, 12, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 4, 16, sizeof(unsigned), &searchval));
+        dd->append(createSingleKeySegmentMonitor(false, 2, 8, sizeof(unsigned), &searchval));
         ASSERT(d->postFilter.length()==3);
         ASSERT(d->segMonitors.length()==0);
         ASSERT(d->postFilter.item(0).getOffset()==8);
@@ -2183,48 +2185,48 @@ protected:
 
         SegMonitorArray q;
         unsigned char searchval;
-        q.append(*createSingleKeySegmentMonitor(false, 'p', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'q', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'p', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'q', 1, &searchval));
         unsigned i;
         for (i = 0; i < 104; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'a', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'b', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'a', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'b', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 103; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'c', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'd', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'c', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'd', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 102; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'e', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'f', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'e', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'f', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 101; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'g', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'h', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'g', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'h', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 3, 'y', 1, &searchval));
         for (i = 0; i < 100; i++)
             stats.noteQuery(q, 0);
 
         q.kill();
-        q.append(*createSingleKeySegmentMonitor(false, 'g', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'x', 1, &searchval));
-        q.append(*createSingleKeySegmentMonitor(false, 'y', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 0, 'g', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 1, 'x', 1, &searchval));
+        q.append(*createSingleKeySegmentMonitor(false, 2, 'y', 1, &searchval));
         for (i = 0; i < 100; i++)
             stats.noteQuery(q, 0);
 

rtl/eclrtl/eclhelper_dyn.cpp

@@ -147,27 +147,27 @@ public:
             MemoryBuffer hibuffer;
             Owned<IStringSet> filterSet = createStringSet(fieldSize);
             deserializeSet(*filterSet, inrec.getMinRecordSize(), fieldType, filter);
-            aindex_t found = filterOffsets.bSearch(fieldOffset, compareOffsets);
-            if (found==NotFound)
+            while (filters.length()<=fieldNum)
             {
-                bool isNew;
-                unsigned insertPos = filterOffsets.bAdd(fieldOffset, compareOffsets, isNew);
-                dbgassertex(isNew);
-                filters.add(*filterSet.getClear(), insertPos);
-            }
-            else
-            {
-                filterSet.setown(filters.item(found).unionSet(filterSet));// Debatable - would intersect be more appropriate?
-                filters.replace(*filterSet.getClear(), found);
+                filters.append(nullptr);
+                filterOffsets.append(offsetCalculator.getOffset(filters.length()));
+                filterOffsets.append(offsetCalculator.getSize(filters.length()));
             }
+            IStringSet *prev = filters.item(fieldNum);
+            if (prev)
+                filterSet.setown(prev->unionSet(filterSet)); // Debatable - would intersect be more appropriate?
+            filters.replace(filterSet.getClear(), fieldNum);
+            filterOffsets.replace(fieldOffset, fieldOffset); // MORE - probably refactor this in  a bit
+            filterSizes.replace(fieldSize, fieldSize); // MORE - probably refactor this in  a bit
         }
     }
     void createSegmentMonitors(IIndexReadContext *irc)
     {
         ForEachItemIn(idx, filters)
         {
-            IStringSet &filter = filters.item(idx);
-            irc->append(createKeySegmentMonitor(false, LINK(&filter), filterOffsets.item(idx), filter.getSize()));
+            IStringSet *filter = filters.item(idx);
+            if (filter)
+                irc->append(createKeySegmentMonitor(false, LINK(filter), idx, filterOffsets.item(idx), filter->getSize()));
         }
     }
     void createSegmentMonitorsWithWild(IIndexReadContext *irc, unsigned keySize)
@@ -175,20 +175,27 @@ public:
         unsigned lastOffset = 0;
         ForEachItemIn(idx, filters)
         {
-            IStringSet &filter = filters.item(idx);
+            IStringSet *filter = filters.item(idx);
             unsigned offset =  filterOffsets.item(idx);
-            unsigned size = filter.getSize();
-            if (offset > lastOffset)
-                irc->append(createWildKeySegmentMonitor(lastOffset, offset-lastOffset));
-            irc->append(createKeySegmentMonitor(false, LINK(&filter), offset, size));
-            lastOffset = offset+size;
+            unsigned size =  filterSizes.item(idx);
+            if (filter)
+            {
+                assertex(size = filter->getSize());
+                irc->append(createKeySegmentMonitor(false, LINK(filter), idx, offset, size));
+            }
+            else
+                irc->append(createWildKeySegmentMonitor(idx, offset, size)); // MORE - move this logic to irc::append ?
         }
+        // MORE - trailing wild needs adding
+        /*
         if (keySize > lastOffset)
             irc->append(createWildKeySegmentMonitor(lastOffset, keySize-lastOffset));
+            */
     }
 protected:
-    IArrayOf<IStringSet> filters;
+    IPointerArrayOf<IStringSet> filters;
     UnsignedArray filterOffsets;
+    UnsignedArray filterSizes;
     const RtlRecord &inrec;
 };
 

rtl/eclrtl/rtlkey.hpp

@@ -28,9 +28,7 @@ enum KeySegmentMonitorSerializeType
     KSMST_SINGLEBIGSIGNEDKEYSEGMENTMONITOR,
     KSMST_SINGLELITTLESIGNEDKEYSEGMENTMONITOR,
     KSMST_CSINGLELITTLEKEYSEGMENTMONITOR,
-    KSMST_DUMMYKEYSEGMENTMONITOR,
     KSMST_OVERRIDEABLEKEYSEGMENTMONITOR,
-    KSMST_VAROFFSETKEYSEGMENTMONITOR,
     KSMST_max
 };
 
@@ -80,48 +78,39 @@ ECLRTL_API int memcmplittlesigned(const void *l, const void *r, unsigned size);
 
 class RtlRow;
 
-// In the parameter names below, "expanded" means it's a pointer to a "key buffer" value, while
-// "raw" means it's a pointer to a disk record. For indexes, the two are the same, but for
-// disk files where we may be using segmonitors as a way to serialize remote filters and to build on-the-fly indexes,
-// the distinction is significant.
-// To support varoffset segmonitors, all "raw" values are passed as const RtlRow *. We do not support varoffset segmonitors in
-// on-the-fly in-memory indexes, meaning that the functions taking "expanded" params do not need to support offset translation
-// and the varoffset segmonitor does not implement these functions.
-
 interface IKeySegmentMonitor : public IInterface
 {
 public:
     virtual bool increment(void * expandedRow) const = 0;
     virtual void setLow(void * expandedRow) const = 0;
+    virtual void setHigh(void * expandedRow) const = 0;
     virtual void endRange(void * expandedRow) const = 0;
+
     virtual bool matchesBuffer(const void * expandedRow) const = 0;
     virtual bool matches(const RtlRow * rawRow) const = 0;
-    virtual IKeySegmentMonitor *merge(IKeySegmentMonitor *with) const = 0;  // merge with adjacent, if possible
-    virtual IKeySegmentMonitor *combine(const IKeySegmentMonitor *with) const = 0; // combine with overlapping (compulsory)
-    virtual IKeySegmentMonitor * split(unsigned splitSize) = 0;
+
     virtual bool isWild() const = 0;
+    virtual unsigned getFieldIdx() const = 0;
     virtual unsigned getOffset() const = 0;
     virtual unsigned getSize() const = 0;
-    virtual void *queryValue() const = 0;
-    virtual int docompare(const void * expandedLeft, const void * rawRight) const = 0;
-//  virtual unsigned getFlags() const = 0;
     virtual bool isEmpty() const = 0;
     virtual bool equivalentTo(const IKeySegmentMonitor &other) const = 0;
     virtual bool isSigned() const = 0;
     virtual bool isLittleEndian() const = 0;
-    virtual int docompareraw(const void * rawLeft, const void * rawRight) const = 0; // NOTE - no RtlRow version since only used for in-memory index builds
+
+    virtual int docompare(const void * expandedLeft, const void * rawRight) const = 0;
     virtual unsigned queryHashCode() const = 0;
     virtual bool isWellKeyed() const = 0;
-    virtual bool setOffset(unsigned _offset) = 0;
     virtual bool isOptional() const = 0;
 
-    virtual void setHigh(void * expandedRow) const = 0;
     virtual bool isSimple() const = 0;
     virtual void copy(void *expandedRow, const void *rawRow) const = 0;
     virtual MemoryBuffer &serialize(MemoryBuffer &mb) const = 0;
     virtual KeySegmentMonitorSerializeType serializeType() const = 0;
     virtual IKeySegmentMonitor *clone() const = 0;
     virtual unsigned numFieldsRequired() const = 0;
+
+    virtual bool setOffset(unsigned _offset) = 0;  // Used by old record layout translator - to be removed at some point
 };
 
 interface IOverrideableKeySegmentMonitor  : public IKeySegmentMonitor
@@ -148,19 +137,6 @@ public:
     virtual void setMergeBarrier(unsigned offset) = 0;
 };
 
-interface IKeySegmentOffsetTranslator : public IInterface
-{
-    virtual const void * getSegmentBase(const void * row) const = 0;
-    virtual const char * queryFactoryName() const = 0;
-};
-
-interface IKeySegmentFormatTranslator : public IInterface
-{
-    virtual void extractField(void * target, const void * row) const = 0;
-    virtual const char * queryFactoryName() const = 0;
-    virtual unsigned queryHashCode() const = 0;
-};
-
 ECLRTL_API IStringSet *createRtlStringSet(size32_t size);
 ECLRTL_API IStringSet *createRtlStringSetEx(size32_t size, bool bigEndian, bool isSigned);
 ECLRTL_API IStringSet *createRtlStringValue(size32_t size, const char * value);
@@ -169,21 +145,18 @@ ECLRTL_API IStringSet *createRtlStringValue(size32_t size, const char * value);
 ECLRTL_API IStringSet *rtlUnionSet(IStringSet * lhs, IStringSet * rhs);
 ECLRTL_API IStringSet *rtlIntersectSet(IStringSet * lhs, IStringSet * rhs);
 
-ECLRTL_API IKeySegmentMonitor *createKeySegmentMonitor(bool optional, IStringSet *set, unsigned _offset, unsigned _size);
-ECLRTL_API IKeySegmentMonitor *createEmptyKeySegmentMonitor(bool optional, unsigned _offset, unsigned _size);
-ECLRTL_API IKeySegmentMonitor *createWildKeySegmentMonitor(unsigned _offset, unsigned _size);
-ECLRTL_API IKeySegmentMonitor *createDummyKeySegmentMonitor(unsigned _offset, unsigned _size, bool isSigned, bool isLittleEndian);
-ECLRTL_API IKeySegmentMonitor *createSingleKeySegmentMonitor(bool optional, unsigned _offset, unsigned _size, const void * value);
-ECLRTL_API IOverrideableKeySegmentMonitor *createOverrideableKeySegmentMonitor(IKeySegmentMonitor *base);
-ECLRTL_API IKeySegmentMonitor *createSingleBigSignedKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
-ECLRTL_API IKeySegmentMonitor *createSingleLittleSignedKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
-ECLRTL_API IKeySegmentMonitor *createSingleLittleKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
+// Various types of KeySegmentMonitor, called from generated code
 
-//takes over ownership of base
-ECLRTL_API IKeySegmentMonitor *createNewVarOffsetKeySegmentMonitor(IKeySegmentMonitor * base, unsigned fieldIdx);
+ECLRTL_API IKeySegmentMonitor *createKeySegmentMonitor(bool optional, IStringSet *set, unsigned _fieldIdx, unsigned _offset, unsigned _size);
+ECLRTL_API IKeySegmentMonitor *createEmptyKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned _offset, unsigned _size);
+ECLRTL_API IKeySegmentMonitor *createWildKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, unsigned _size);
+ECLRTL_API IKeySegmentMonitor *createDummyKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, unsigned _size, bool isSigned, bool isLittleEndian);
+ECLRTL_API IKeySegmentMonitor *createSingleKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned _offset, unsigned _size, const void * value);
+ECLRTL_API IKeySegmentMonitor *createSingleBigSignedKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned offset, unsigned size, const void * value);
+ECLRTL_API IKeySegmentMonitor *createSingleLittleSignedKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned offset, unsigned size, const void * value);
+ECLRTL_API IKeySegmentMonitor *createSingleLittleKeySegmentMonitor(bool optional, unsigned _fieldIdx, unsigned offset, unsigned size, const void * value);
 
-//takes over ownership of both arguments
-ECLRTL_API IKeySegmentMonitor *createTranslatedKeySegmentMonitor(IKeySegmentMonitor * base, unsigned offset, IKeySegmentFormatTranslator * translator);
+ECLRTL_API IOverrideableKeySegmentMonitor *createOverrideableKeySegmentMonitor(IKeySegmentMonitor *base);
 
 ECLRTL_API IKeySegmentMonitor *deserializeKeySegmentMonitor(MemoryBuffer &mb);
 ECLRTL_API void deserializeSet(IStringSet & set, size32_t minRecordSize, const RtlTypeInfo * fieldType, const char * filter);

system/jhtree/jhtree.cpp

@@ -110,7 +110,7 @@ size32_t SegMonitorList::getSize() const
 void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname)
 {
     size32_t segSize = getSize();
-    if (segSize > keyedSize)
+    if (segSize != keyedSize)
     {
         StringBuffer err;
         err.appendf("Key size mismatch on key %s - key size is %u, expected %u", keyname, keyedSize, getSize());
@@ -118,8 +118,6 @@ void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname)
         EXCLOG(e, err.str());
         throw e;
     }
-    else if (segSize < keyedSize)
-        segMonitors.append(*createWildKeySegmentMonitor(segSize, keyedSize-segSize));
 }
 
 void SegMonitorList::setLow(unsigned segno, void *keyBuffer) const
@@ -215,44 +213,18 @@ unsigned SegMonitorList::lastFullSeg() const
     return ret;
 }
 
-void SegMonitorList::finish()
+void SegMonitorList::finish(unsigned keyedSize) // MORE - probably need to know the record layout here, if wilds are to be added properly.
 {
     if (modified)
     {
-        unsigned len = segMonitors.length();
-        unsigned curOffset = 0;
-        unsigned curSize = 0;
-        if (len)
+        size32_t segSize = getSize();
+        if (segSize < mergeBarrier)
         {
-            IKeySegmentMonitor *current = &segMonitors.item(0);
-            curOffset = current->getOffset();
-            curSize = current->getSize();
-            for (unsigned seg = 1; seg < len; seg++)
-            {
-                IKeySegmentMonitor *next = &segMonitors.item(seg);
-                unsigned nextOffset = next->getOffset();
-                unsigned nextSize = next->getSize();
-                assertex(curOffset + curSize == nextOffset) ;
-                IKeySegmentMonitor *merged = NULL;
-                if (nextOffset != mergeBarrier)
-                    merged = current->merge(next);
-                if (merged)
-                {
-                    current = merged;
-                    segMonitors.replace(*current, seg-1);
-                    segMonitors.remove(seg);
-                    curSize += nextSize;
-                    seg--;
-                    len--;
-                }
-                else
-                {
-                    current = next;
-                    curOffset = nextOffset;
-                    curSize = nextSize;
-                }
-            }
+            segMonitors.append(*createWildKeySegmentMonitor(0, segSize, mergeBarrier-segSize));  // MORE - field number is needed, but that requires layout info
+            segSize = mergeBarrier;
         }
+         if (segSize < keyedSize)
+            segMonitors.append(*createWildKeySegmentMonitor(0, segSize, keyedSize-segSize));  // MORE - field number is needed, but that requires layout info
         recalculateCache();
         modified = false;
     }
@@ -294,81 +266,40 @@ void SegMonitorList::serialize(MemoryBuffer &mb) const
 void SegMonitorList::append(IKeySegmentMonitor *segment)
 {
     modified = true;
+    unsigned fieldIdx = segment->getFieldIdx();
     unsigned offset = segment->getOffset();
     unsigned size = segment->getSize();
-    if (mergeBarrier)
+#if 0
+    // Eventually we will want something like this:
+    while (segMonitors.length() < fieldIdx)
     {
-        // check if we need to split
-        if (offset < mergeBarrier && offset+size > mergeBarrier)
-        {
-            IKeySegmentMonitor *split = segment->split(mergeBarrier - offset);
-            assertex(split);
-            append(split);   // note - recursive call
-            append(segment);
-            return;
-        }
+        unsigned idx = segMonitors.length();
+        IKeySegmentMonitor * before = createWildKeySegmentMonitor(idx, field(idx));
     }
+#else
+    // But as we don't have field info yet, and we don't care about it yet, this will do
+    unsigned lastOffset = 0;
     if (segMonitors.length())
     {
         unsigned lpos = segMonitors.length()-1;
         IKeySegmentMonitor &last = segMonitors.item(lpos);
-        if (last.getOffset() + last.getSize() == offset)
+        lastOffset = last.getOffset() + last.getSize();
+    }
+    assertex(lastOffset <= offset);
+    if (lastOffset < offset && needWild)
+    {
+        if (mergeBarrier && lastOffset < mergeBarrier && offset > mergeBarrier)
         {
-            segMonitors.append(*segment);
+            segMonitors.append(*createWildKeySegmentMonitor(0, lastOffset, mergeBarrier-lastOffset));
+            segMonitors.append(*createWildKeySegmentMonitor(0, mergeBarrier, offset-mergeBarrier));
         }
         else
         {
-            // we need to combine new segmonitor with existing segmonitors
-            assertex (offset + size <= last.getOffset() + last.getSize());   // no holes should ever be created...
-            if (!segment->isWild())  // X and wild is always X
-            {
-                // Find segmonitor that overlaps the new one, splitting if necessary
-                unsigned idx = 0;
-                while (segMonitors.isItem(idx))
-                {
-                    IKeySegmentMonitor &existing = segMonitors.item(idx);
-                    unsigned existingOffset = existing.getOffset();
-                    unsigned existingSize = existing.getSize();
-                    if (existingOffset <= offset && existingOffset+existingSize > offset)
-                    {
-                        // There is some overlap ...
-                        if (existingOffset < offset)
-                        {
-                            // split existing at start
-                            IKeySegmentMonitor *splitExisting = existing.split(offset-existingOffset);
-                            segMonitors.add(*splitExisting, idx);
-                            // now loop around to handle second half of existing
-                        }
-                        else if (size > existingSize)
-                        {
-                            // split new, merge first half into existing, and loop around to merge second half
-                            Owned<IKeySegmentMonitor> splitNew = segment->split(existingSize);
-                            IKeySegmentMonitor *combined = existing.combine(splitNew);
-                            segMonitors.replace(*combined, idx);
-                        }
-                        else if (size < existingSize)
-                        {
-                            // split existing, then merge new into first half
-                            Owned<IKeySegmentMonitor> split = existing.split(size); // leaves existing in the array as the tail
-                            IKeySegmentMonitor *combined = segment->combine(split);
-                            segMonitors.add(*combined, idx);
-                            break;
-                        }
-                        else // perfect overlap
-                        {
-                            IKeySegmentMonitor *combined = segment->combine(&existing);
-                            segMonitors.add(*combined, idx);
-                            break;
-                        }
-                    }
-                    idx++;
-                }
-            }
-            segment->Release();
+            segMonitors.append(*createWildKeySegmentMonitor(0, lastOffset, offset-lastOffset));
         }
     }
-    else
-        segMonitors.append(*segment);
+#endif
+    segMonitors.append(*segment);
 }
 
 bool SegMonitorList::matched(void *keyBuffer, unsigned &lastMatch) const
@@ -553,7 +484,7 @@ protected:
 public:
     IMPLEMENT_IINTERFACE;
 
-    CKeyLevelManager(IKeyIndex * _key, IContextLogger *_ctx)
+    CKeyLevelManager(IKeyIndex * _key, IContextLogger *_ctx) : segs(true)
     {
         ctx = _ctx;
         numsegs = 0;
@@ -653,8 +584,7 @@ public:
             {
                 started = true;
                 numsegs = segs.ordinality();
-                if (numsegs)
-                    segs.checkSize(keyedSize, keyName.get());
+                segs.checkSize(keyedSize, keyName.get());
             }
             if (!crappyHack)
             {
@@ -668,7 +598,7 @@ public:
 
     virtual void releaseSegmentMonitors()
     {
-        segs.segMonitors.kill();    
+        segs.reset();
         started = false;
         if(layoutTransSegCtx)
             layoutTransSegCtx->reset();
@@ -982,6 +912,7 @@ public:
 
     virtual void finishSegmentMonitors()
     {
+        segs.finish(keyedSize);
         if(transformSegs)
         {
             layoutTrans->createDiskSegmentMonitors(*layoutTransSegCtx, segs);
@@ -993,7 +924,6 @@ public:
                 throw MakeStringExceptionDirect(0, err.str());
             }
         }
-        segs.finish();
     }
 };
 
@@ -2419,7 +2349,6 @@ class CKeyMerger : public CKeyLevelManager
             }
             IKeySegmentMonitor *override = createOverrideableKeySegmentMonitor(LINK(&seg));
             segs.segMonitors.replace(*override, idx);
-
         }
         if (sortFromSeg == -1)
             assertex(!"Attempting to sort from offset that is not on a segment boundary"); // MORE - can use the information that we have earlier to make sure not merged
@@ -2934,7 +2863,11 @@ public:
     {
         CKeyLevelManager::finishSegmentMonitors();
         if (sortFieldOffset)
+        {
+            if (keyedSize)
+                segs.checkSize(keyedSize, "[merger]"); // Ensures trailing KSM is setup
             replicateForTrailingSort();
+        }
     }
 };
 
@@ -3030,14 +2963,14 @@ class IKeyManagerTest : public CppUnit::TestFixture
             Owned<IStringSet> sset1 = createStringSet(7);
             sset1->addRange("0000003", "0000003");
             sset1->addRange("0000005", "0000006");
-            tlk1->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
+            tlk1->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
             Owned<IStringSet> sset2 = createStringSet(3);
             sset2->addRange("010", "010");
             sset2->addRange("030", "033");
             Owned<IStringSet> sset3 = createStringSet(3);
             sset3->addRange("999", "XXX");
             sset3->addRange("000", "002");
-            tlk1->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
+            tlk1->append(createKeySegmentMonitor(false, sset2.getLink(), 1, 7, 3));
             tlk1->finishSegmentMonitors();
 
             tlk1->reset();
@@ -3066,8 +2999,8 @@ class IKeyManagerTest : public CppUnit::TestFixture
             Owned <IKeyManager> tlk2 = createKeyMerger(NULL, 7, NULL);
             tlk2->setKey(keyset);
             tlk2->deserializeCursorPos(mb);
-            tlk2->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
-            tlk2->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
+            tlk2->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
+            tlk2->append(createKeySegmentMonitor(false, sset2.getLink(), 1, 7, 3));
             tlk2->finishSegmentMonitors();
             tlk2->reset(true);
             ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000003032", 10)==0);
@@ -3081,8 +3014,8 @@ class IKeyManagerTest : public CppUnit::TestFixture
 
             Owned <IKeyManager> tlk3 = createKeyMerger(NULL, 7, NULL);
             tlk3->setKey(keyset);
-            tlk3->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
-            tlk3->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
+            tlk3->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
+            tlk3->append(createKeySegmentMonitor(false, sset2.getLink(), 1, 7, 3));
             tlk3->finishSegmentMonitors();
             tlk3->reset(false);
             ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000003010", 10)==0);
@@ -3094,8 +3027,8 @@ class IKeyManagerTest : public CppUnit::TestFixture
 
             Owned <IKeyManager> tlk4 = createKeyMerger(NULL, 7, NULL);
             tlk4->setKey(keyset);
-            tlk4->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 7));
-            tlk4->append(createKeySegmentMonitor(false, sset3.getLink(), 7, 3));
+            tlk4->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
+            tlk4->append(createKeySegmentMonitor(false, sset3.getLink(), 1, 7, 3));
             tlk4->finishSegmentMonitors();
             tlk4->reset(false);
             ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003000", 10)==0);
@@ -3219,18 +3152,18 @@ protected:
             Owned <IKeyManager> tlk1 = createLocalKeyManager(index1, NULL);
             Owned<IStringSet> sset1 = createStringSet(10);
             sset1->addRange("0000000001", "0000000100");
-            tlk1->append(createKeySegmentMonitor(false, sset1.getClear(), 0, 10));
+            tlk1->append(createKeySegmentMonitor(false, sset1.getClear(), 0, 0, 10));
             tlk1->finishSegmentMonitors();
             tlk1->reset();
 
             Owned <IKeyManager> tlk1a = createLocalKeyManager(index1, NULL);
             Owned<IStringSet> sset1a = createStringSet(8);
             sset1a->addRange("00000000", "00000001");
-            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 0, 8));
-            tlk1a->append(createKeySegmentMonitor(false, NULL, 8, 1));
+            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 0, 0, 8));
+            tlk1a->append(createKeySegmentMonitor(false, NULL, 1, 8, 1));
             sset1a.setown(createStringSet(1));
             sset1a->addRange("0", "1");
-            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 9, 1));
+            tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 2, 9, 1));
             tlk1a->finishSegmentMonitors();
             tlk1a->reset();
 
@@ -3259,7 +3192,7 @@ protected:
             Owned<IStringSet> sset2 = createStringSet(10);
             sset2->addRange("0000000001", "0000000100");
             ASSERT(sset2->numValues() == 65536);
-            tlk2->append(createKeySegmentMonitor(false, sset2.getClear(), 0, 10));
+            tlk2->append(createKeySegmentMonitor(false, sset2.getClear(), 0, 0, 10));
             tlk2->finishSegmentMonitors();
             tlk2->reset();
 
@@ -3273,7 +3206,7 @@ protected:
                 tlk3.setown(createKeyMerger(NULL, 0, NULL));
                 tlk3->setKey(both);
                 sset3->addRange("0000000001", "0000000100");
-                tlk3->append(createKeySegmentMonitor(false, sset3.getClear(), 0, 10));
+                tlk3->append(createKeySegmentMonitor(false, sset3.getClear(), 0, 0, 10));
                 tlk3->finishSegmentMonitors();
                 tlk3->reset();
             }
@@ -3282,7 +3215,7 @@ protected:
             Owned<IStringSet> sset2a = createStringSet(10);
             sset2a->addRange("0000000048", "0000000048");
             ASSERT(sset2a->numValues() == 1);
-            tlk2a->append(createKeySegmentMonitor(false, sset2a.getClear(), 0, 10));
+            tlk2a->append(createKeySegmentMonitor(false, sset2a.getClear(), 0, 0, 10));
             tlk2a->finishSegmentMonitors();
             tlk2a->reset();
 
@@ -3290,14 +3223,14 @@ protected:
             Owned<IStringSet> sset2b = createStringSet(10);
             sset2b->addRange("0000000047", "0000000049");
             ASSERT(sset2b->numValues() == 3);
-            tlk2b->append(createKeySegmentMonitor(false, sset2b.getClear(), 0, 10));
+            tlk2b->append(createKeySegmentMonitor(false, sset2b.getClear(), 0, 0, 10));
             tlk2b->finishSegmentMonitors();
             tlk2b->reset();
 
             Owned <IKeyManager> tlk2c = createLocalKeyManager(index2, NULL);
             Owned<IStringSet> sset2c = createStringSet(10);
             sset2c->addRange("0000000047", "0000000047");
-            tlk2c->append(createKeySegmentMonitor(false, sset2c.getClear(), 0, 10));
+            tlk2c->append(createKeySegmentMonitor(false, sset2c.getClear(), 0, 0, 10));
             tlk2c->finishSegmentMonitors();
             tlk2c->reset();
 

system/jhtree/jhtree.hpp

@@ -160,9 +160,10 @@ class jhtree_decl SegMonitorList : implements IInterface, implements IIndexReadC
     unsigned cachedLRS;
     unsigned mergeBarrier;
     bool modified;
+    bool needWild;
 public:
     IMPLEMENT_IINTERFACE;
-    inline SegMonitorList() { reset(); }
+    inline SegMonitorList(bool _needWild) : needWild(_needWild) { reset(); }
     IArrayOf<IKeySegmentMonitor> segMonitors;
 
     void reset();
@@ -179,7 +180,7 @@ public:
 
     void checkSize(size32_t keyedSize, char const * keyname);
     void recalculateCache();
-    void finish();
+    void finish(size32_t keyedSize);
     void deserialize(MemoryBuffer &mb);
     void serialize(MemoryBuffer &mb) const;
 

testing/regress/ecl/bug12130.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/bug5236.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/canmatch.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/childindex.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/countindex.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/diskread.ecl

@@ -104,7 +104,7 @@ output(DG_FlatFile_pl0(KEYED(DG_firstname='CLAIRE')));
 output(DG_FlatFileEvens_pl0(KEYED(DG_firstname='CLAIRE')));
 //output(DG_CSVFile_pl0(KEYED(DG_firstname='CLAIRE')));
 //output(DG_XMLFile_pl0(KEYED(DG_firstname='CLAIRE')));
-output(DG_VarFile_pl0(KEYED(DG_firstname='CLAIRE')));
+//output(DG_VarFile_pl0(KEYED(DG_firstname='CLAIRE')));
 
 /*
 // now repeat with preload(n)....
@@ -145,7 +145,7 @@ output(DG_FlatFile_pl2(KEYED(DG_firstname='CLAIRE')));
 output(DG_FlatFileEvens_pl2(KEYED(DG_firstname='CLAIRE')));
 //output(DG_CSVFile_pl2(KEYED(DG_firstname='CLAIRE')));
 //output(DG_XMLFile_pl2(KEYED(DG_firstname='CLAIRE')));
-output(DG_VarFile_pl2(KEYED(DG_firstname='CLAIRE')));
+//output(DG_VarFile_pl2(KEYED(DG_firstname='CLAIRE')));
 
 */
 

testing/regress/ecl/fetch.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/indexagg.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/indexread.ecl

@@ -19,7 +19,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/indexread2.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/indexread4.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/indexread6.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/key/diskread.xml

@@ -106,21 +106,3 @@
  <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>350</filepos></Row>
  <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>375</filepos></Row>
 </Dataset>
-<Dataset name='Result 12'>
- <Row><emptyfield></emptyfield><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><__filepos>624</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><extrafield>BACL                </extrafield><__filepos>653</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><__filepos>702</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><extrafield>BAYLCLAI            </extrafield><__filepos>731</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><__filepos>780</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><extrafield>DOCL                </extrafield><__filepos>809</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><__filepos>858</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><extrafield>DOLSCLAI            </extrafield><__filepos>887</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><__filepos>936</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><extrafield>BICL                </extrafield><__filepos>965</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><__filepos>1014</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><extrafield>BILLCLAI            </extrafield><__filepos>1043</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><__filepos>1092</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><extrafield>SMCL                </extrafield><__filepos>1121</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><__filepos>1170</__filepos></Row>
- <Row><emptyfield></emptyfield><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><extrafield>SMITCLAI            </extrafield><__filepos>1199</__filepos></Row>
-</Dataset>

testing/regress/ecl/key/layouttrans.xml

@@ -1,60 +1,3 @@
 <Dataset name='Result 1'>
- <Row><fname>Frank          </fname><lname>Smith                    </lname></Row>
- <Row><fname>Fred           </fname><lname>Smith                    </lname></Row>
- <Row><fname>Nancy          </fname><lname>Smith                    </lname></Row>
- <Row><fname>Sue            </fname><lname>Smith                    </lname></Row>
- <Row><fname>Zeek           </fname><lname>Smith                    </lname></Row>
-</Dataset>
-<Dataset name='Result 2'>
- <Row><fname>Frank          </fname><lname>Smith                    </lname></Row>
- <Row><fname>Fred           </fname><lname>Smith                    </lname></Row>
- <Row><fname>Nancy          </fname><lname>Smith                    </lname></Row>
- <Row><fname>Sue            </fname><lname>Smith                    </lname></Row>
- <Row><fname>Zeek           </fname><lname>Smith                    </lname></Row>
-</Dataset>
-<Dataset name='Result 3'>
- <Row><fname>Frank          </fname><lname>Smith                    </lname></Row>
- <Row><fname>Fred           </fname><lname>Smith                    </lname></Row>
- <Row><fname>Nancy          </fname><lname>Smith                    </lname></Row>
- <Row><fname>Sue            </fname><lname>Smith                    </lname></Row>
- <Row><fname>Zeek           </fname><lname>Smith                    </lname></Row>
-</Dataset>
-<Dataset name='Result 4'>
- <Row><fname>Zeek           </fname><lname>Smith                    </lname></Row>
-</Dataset>
-<Dataset name='Result 5'>
- <Row><fname>Zeek           </fname><lname>Smith                    </lname></Row>
-</Dataset>
-<Dataset name='Result 6'>
- <Row><fname>Zeek           </fname><lname>Smith                    </lname></Row>
-</Dataset>
-<Dataset name='Result 7'>
- <Row><fname>Jane           </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Joe            </fname><lname>Anderson                 </lname></Row>
- <Row><fname>John           </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Larry          </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Sue            </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Jane           </fname><lname>Doe                      </lname></Row>
- <Row><fname>Joe            </fname><lname>Doe                      </lname></Row>
- <Row><fname>John           </fname><lname>Doe                      </lname></Row>
-</Dataset>
-<Dataset name='Result 8'>
- <Row><fname>Jane           </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Joe            </fname><lname>Anderson                 </lname></Row>
- <Row><fname>John           </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Larry          </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Sue            </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Jane           </fname><lname>Doe                      </lname></Row>
- <Row><fname>Joe            </fname><lname>Doe                      </lname></Row>
- <Row><fname>John           </fname><lname>Doe                      </lname></Row>
-</Dataset>
-<Dataset name='Result 9'>
- <Row><fname>Jane           </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Joe            </fname><lname>Anderson                 </lname></Row>
- <Row><fname>John           </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Larry          </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Sue            </fname><lname>Anderson                 </lname></Row>
- <Row><fname>Jane           </fname><lname>Doe                      </lname></Row>
- <Row><fname>Joe            </fname><lname>Doe                      </lname></Row>
- <Row><fname>John           </fname><lname>Doe                      </lname></Row>
+ <Row><Result_1>Temporarily disabled</Result_1></Row>
 </Dataset>

testing/regress/ecl/keyed_denormalize.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/keyed_join.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/keyed_join2.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/layouttrans.ecl

@@ -15,9 +15,12 @@
     limitations under the License.
 ############################################################################## */
 
-//version multiPart=true,version=1
-//version multiPart=true,version=2
-//version multiPart=true,version=3
+#if (True)
+OUTPUT('Temporarily disabled');
+#else
+//noversion multiPart=true,version=1
+//noversion multiPart=true,version=2
+//noversion multiPart=true,version=3
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);
@@ -61,3 +64,4 @@ SEQUENTIAL(
     OUTPUT(SORT(JOIN(ds, DG_FetchIndex1Alt1, LEFT.Lname = RIGHT.Lname), record), {Fname, Lname}),
     OUTPUT(SORT(JOIN(ds, DG_FetchIndex1Alt2, LEFT.Lname = RIGHT.Lname), record), {Fname, Lname})
 );
+#end

testing/regress/ecl/tablecount.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);

testing/regress/ecl/tablecount2.ecl

@@ -20,7 +20,7 @@
 //version multiPart=false
 //version multiPart=true
 //version multiPart=true,useLocal=true
-//version multiPart=true,useTranslation=true,nothor
+//noversion multiPart=true,useTranslation=true,nothor
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, true);