Merge pull request #11100 from richardkchapman/jhtree-filter

HPCC-19584 Add support for new field filters into JHTree

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/fileview2/fvidxsource.cpp

@@ -432,7 +432,7 @@ bool IndexDataSource::addFilter(unsigned column, unsigned matchLen, unsigned siz
 
 void IndexDataSource::applyFilter()
 {
-    manager.setown(createLocalKeyManager(diskRecordMeta->queryRecordAccessor(true), tlk, NULL));
+    manager.setown(createLocalKeyManager(diskRecordMeta->queryRecordAccessor(true), tlk, NULL, false));
     ForEachItemIn(i, values)
     {
         IStringSet & cur = values.item(i);

common/thorhelper/thorxmlwrite.cpp

@@ -121,137 +121,6 @@ void CommonFieldProcessor::processEndRow(const RtlFieldInfo * field)
 
 
 //=============================================================================================
-// MORE - this function should probably move into IIndexReadContext interface rather than leaking ordinality() and item() out of that interface just for me
-void printKeyedValues(StringBuffer &out, IIndexReadContext *segs, IOutputMetaData *rowMeta)
-{
-    unsigned totalKeyedSize = 0;
-    unsigned numSegs = segs->ordinality();
-    while (numSegs)
-    {
-        IKeySegmentMonitor &seg = *segs->item(numSegs-1);
-        if (!seg.isWild())
-        {
-            totalKeyedSize = seg.getOffset() + seg.getSize();
-            break;
-        }
-        numSegs--;
-    }
-    if (numSegs)
-    {
-        byte *tempRow = (byte *) alloca(totalKeyedSize);
-        byte *savedRow = (byte *) alloca(totalKeyedSize);
-        const RtlFieldInfo * const *fields = rowMeta->queryTypeInfo()->queryFields();
-        unsigned fieldOffset = 0;
-        bool inKeyed = false;
-        bool inWild = false;
-        for (unsigned segNo = 0; segNo < numSegs; segNo++)
-        {
-            IKeySegmentMonitor &seg = *segs->item(segNo);
-            unsigned segOffset = seg.getOffset();
-            unsigned segSize = seg.getSize();
-            while (fieldOffset < segOffset + segSize) // This is trying to cope with the combined case but not sure it completely does
-            {
-                assertex(fields[0]->type->isFixedSize());
-                unsigned curFieldSize = fields[0]->type->size(NULL, NULL);
-                if (seg.isWild())
-                {
-                    if (!inWild)
-                    {
-                        if (inKeyed)
-                        {
-                            out.append("),");
-                            inKeyed = false;
-                        }
-                        out.append("WILD(");
-                        inWild = true;
-                    }
-                    else
-                        out.append(',');
-                    out.append(fields[0]->name);
-                }
-                else
-                {
-                    StringBuffer setValues;
-                    CommonFieldProcessor setProcessor(setValues, true);
-                    unsigned numValues = 0;
-                    unsigned subStringLength = 0;
-                    if (!seg.isEmpty())
-                    {
-                        seg.setLow(tempRow);
-                        for (;;)
-                        {
-                            if (numValues)
-                                setValues.append(",");
-                            memcpy(savedRow+segOffset, tempRow+segOffset, segSize);
-                            seg.endRange(tempRow);
-                            if (memcmp(savedRow+segOffset, tempRow+segOffset, segSize) != 0)
-                            {
-                                // Special case - if they differ only in trailing values that are 0 vs 0xff, then it's a substring match...
-                                if (numValues==0 && (fields[0]->type->fieldType & (RFTMkind | RFTMebcdic)) == type_string)
-                                {
-                                    unsigned pos;
-                                    for (pos = 0; pos < segSize; pos++)
-                                    {
-                                        if (savedRow[segOffset+pos] != tempRow[segOffset+pos])
-                                            break;
-                                    }
-                                    subStringLength = pos;
-                                    for (; pos < segSize; pos++)
-                                    {
-                                        if (savedRow[segOffset+pos] != 0 || tempRow[segOffset+pos] != 0xff)
-                                        {
-                                            subStringLength = 0;
-                                            break;
-                                        }
-                                    }
-                                }
-                                fields[0]->process(savedRow + fieldOffset, tempRow, setProcessor);
-                                setValues.append("..");
-                                fields[0]->process(tempRow + fieldOffset, tempRow, setProcessor);
-                                numValues+=2;
-                            }
-                            else
-                            {
-                                fields[0]->process(tempRow + fieldOffset, tempRow, setProcessor);
-                                numValues++;
-                            }
-                            if (!seg.increment(tempRow))
-                                break;
-                        }
-                    }
-                    if (!inKeyed)
-                    {
-                        if (inWild)
-                        {
-                            out.append("),");
-                            inWild = false;
-                        }
-                        out.append("KEYED(");
-                        inKeyed = true;
-                    }
-                    else
-                        out.append(',');
-                    out.append(fields[0]->name);
-                    if (numValues==1)
-                        out.append("=").append(setValues);
-                    else if (subStringLength)
-                        out.appendf("[1..%d]='", subStringLength).append(subStringLength, (char *) savedRow+fieldOffset).append("'");
-                    else
-                        out.append(" IN [").append(setValues).append("]");
-                }
-                fieldOffset += curFieldSize;
-                fields++;
-                if (!fields[0])
-                    break;
-            }
-        }
-        if (inKeyed || inWild)
-            out.append(")");
-    }
-    else
-        out.append("UNKEYED");
-}
-
 
 extern thorhelper_decl void convertRowToXML(size32_t & lenResult, char * & result, IOutputMetaData & info, const void * row, unsigned flags)
 {

ecl/hqlcpp/hqlckey.cpp

@@ -1433,7 +1433,6 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
         flags.append("|JFnewfilters");
     if (flags.length())
         doBuildUnsignedFunction(instance->classctx, "getJoinFlags", flags.str()+1);
-
     //Fetch flags
     flags.clear();
     if (info.isFullJoin())
@@ -1551,7 +1550,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx
         flags.append("|KDFvarindexfilename");
     if (dynamic)
         flags.append("|KDFdynamicindexfilename");
-
+    if (options.createValueSets)
+        flags.append("|KDFnewfilters");
 
     if (flags.length())
         doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);

ecl/hqlcpp/hqlcpp.cpp

@@ -1808,7 +1808,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.timeTransforms,"timeTransforms", false),
         DebugOption(options.reportDFSinfo,"reportDFSinfo", 0),
         DebugOption(options.useGlobalCompareClass,"useGlobalCompareClass", false),
-        DebugOption(options.createValueSets,"createValueSets", false),
+        DebugOption(options.createValueSets,"createValueSets", true),
         DebugOption(options.implicitKeyedDiskFilter,"implicitKeyedDiskFilter", false),
         DebugOption(options.addDefaultBloom,"addDefaultBloom", true),
         DebugOption(options.newDiskReadMapping, "newDiskReadMapping", true),

ecl/hqlcpp/hqlsource.cpp

@@ -3856,6 +3856,8 @@ void IndexReadBuilderBase::buildFlagsMember(IHqlExpression * expr)
     if (!nameExpr->isConstant()) flags.append("|TIRvarfilename");
     if (translator.hasDynamicFilename(tableExpr)) flags.append("|TIRdynamicfilename");
     if (requiresOrderedMerge) flags.append("|TIRorderedmerge");
+    if (translator.queryOptions().createValueSets)
+        flags.append("|TIRnewfilters");
 
     if (flags.length())
         translator.doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);

ecl/hthor/hthor.ipp

@@ -2318,8 +2318,6 @@ public:
 
     //interface IIndexReadContext
     virtual void append(IKeySegmentMonitor *segment) override { throwUnexpected(); }
-    virtual unsigned ordinality() const override { throwUnexpected(); }
-    virtual IKeySegmentMonitor *item(unsigned idx) const override { throwUnexpected();  }
     virtual void append(FFoption option, const IFieldFilter * filter) override;
 
 protected:

ecl/hthor/hthorkey.cpp

@@ -425,7 +425,7 @@ bool CHThorIndexReadActivityBase::doPreopenLimitFile(unsigned __int64 & count, u
         {
             Owned<IKeyIndex> tlk = openKeyFile(df->queryPart(num));
             verifyIndex(tlk);
-            Owned<IKeyManager> tlman = createLocalKeyManager(eclKeySize.queryRecordAccessor(true), tlk, NULL);
+            Owned<IKeyManager> tlman = createLocalKeyManager(eclKeySize.queryRecordAccessor(true), tlk, NULL, helper.hasNewSegmentMonitors());
             initManager(tlman, true);
             while(tlman->lookup(false) && (count<=limit))
             {
@@ -461,7 +461,7 @@ IKeyIndex * CHThorIndexReadActivityBase::doPreopenLimitPart(unsigned __int64 & r
         verifyIndex(kidx);
     if (limit != (unsigned) -1)
     {
-        Owned<IKeyManager> kman = createLocalKeyManager(eclKeySize.queryRecordAccessor(true), kidx, NULL);
+        Owned<IKeyManager> kman = createLocalKeyManager(eclKeySize.queryRecordAccessor(true), kidx, NULL, helper.hasNewSegmentMonitors());
         initManager(kman, false);
         result += kman->checkCount(limit-result);
     }
@@ -563,7 +563,7 @@ void CHThorIndexReadActivityBase::initManager(IKeyManager *manager, bool isTlk)
 void CHThorIndexReadActivityBase::initPart()                                    
 { 
     assertex(!keyIndex->isTopLevelKey());
-    klManager.setown(createLocalKeyManager(eclKeySize.queryRecordAccessor(true), keyIndex, NULL));
+    klManager.setown(createLocalKeyManager(eclKeySize.queryRecordAccessor(true), keyIndex, NULL, helper.hasNewSegmentMonitors()));
     initManager(klManager, false);
     callback.setManager(klManager);
 }
@@ -593,7 +593,7 @@ bool CHThorIndexReadActivityBase::firstMultiPart()
     if(!tlk)
         openTlk();
     verifyIndex(tlk);
-    tlManager.setown(createLocalKeyManager(eclKeySize.queryRecordAccessor(true), tlk, NULL));
+    tlManager.setown(createLocalKeyManager(eclKeySize.queryRecordAccessor(true), tlk, NULL, helper.hasNewSegmentMonitors()));
     initManager(tlManager, true);
     nextPartNumber = 0;
     return nextMultiPart();
@@ -834,7 +834,7 @@ bool CHThorIndexReadActivity::nextPart()
 {
     if(keyIndexCache && (seekGEOffset || localSortKey))
     {
-        klManager.setown(createKeyMerger(eclKeySize.queryRecordAccessor(true), keyIndexCache, seekGEOffset, NULL));
+        klManager.setown(createKeyMerger(eclKeySize.queryRecordAccessor(true), keyIndexCache, seekGEOffset, NULL, helper.hasNewSegmentMonitors()));
         keyIndexCache.clear();
         initManager(klManager, false);
         callback.setManager(klManager);
@@ -2752,6 +2752,7 @@ interface IJoinProcessor
     virtual const IDynamicTransform * getLayoutTranslator(IDistributedFile * f) = 0;
     virtual const RtlRecord &queryIndexRecord() = 0;
     virtual void verifyIndex(IDistributedFile * f, IKeyIndex * idx, const IDynamicTransform * trans) = 0;
+    virtual bool hasNewSegmentMonitors() = 0;
 };
 
 class CJoinGroup : implements IInterface, public CInterface
@@ -3138,7 +3139,7 @@ public:
             //Owned<IRecordLayoutTranslator> 
             trans.setown(owner.getLayoutTranslator(&f));
             owner.verifyIndex(&f, index, trans);
-            Owned<IKeyManager> manager = createLocalKeyManager(owner.queryIndexRecord(), index, NULL);
+            Owned<IKeyManager> manager = createLocalKeyManager(owner.queryIndexRecord(), index, NULL, owner.hasNewSegmentMonitors());
             managers.append(*manager.getLink());
         }
         opened = true;
@@ -3177,7 +3178,7 @@ void KeyedLookupPartHandler::openPart()
     if(manager)
         return;
     Owned<IKeyIndex> index = openKeyFile(*part);
-    manager.setown(createLocalKeyManager(owner.queryIndexRecord(), index, NULL));
+    manager.setown(createLocalKeyManager(owner.queryIndexRecord(), index, NULL, owner.hasNewSegmentMonitors()));
     const IDynamicTransform * trans = tlk->queryRecordLayoutTranslator();
     if(trans && !index->isTopLevelKey())
         manager->setLayoutTranslator(trans);
@@ -3257,7 +3258,7 @@ public:
             {
                 Owned<IKeyIndex> index = openKeyFile(f.queryPart(0));
                 owner.verifyIndex(&f, index, trans);
-                manager.setown(createLocalKeyManager(owner.queryIndexRecord(), index, NULL));
+                manager.setown(createLocalKeyManager(owner.queryIndexRecord(), index, NULL, owner.hasNewSegmentMonitors()));
             }
             else
             {
@@ -3270,7 +3271,7 @@ public:
                     parts->addIndex(index.getLink());
                 }
                 owner.verifyIndex(&f, index, trans);
-                manager.setown(createKeyMerger(owner.queryIndexRecord(), parts, 0, nullptr));
+                manager.setown(createKeyMerger(owner.queryIndexRecord(), parts, 0, nullptr, owner.hasNewSegmentMonitors()));
             }
             if(trans)
                 manager->setLayoutTranslator(trans);
@@ -3397,6 +3398,8 @@ public:
 
     virtual bool needsAllocator() const { return true; }
 
+    virtual bool hasNewSegmentMonitors() { return helper.hasNewSegmentMonitors(); }
+
     virtual void ready()        
     { 
         CHThorThreadedActivityBase::ready(); 

roxie/ccd/ccdactivities.cpp

@@ -936,16 +936,6 @@ public:
             postFilter.addFilter(*filter);
     }
 
-    virtual unsigned ordinality() const
-    {
-        throwUnexpected();
-    }
-
-    virtual IKeySegmentMonitor *item(unsigned idx) const
-    {
-        throwUnexpected();
-    }
-
     virtual void abort() 
     {
         CRoxieSlaveActivity::abort();
@@ -2432,6 +2422,7 @@ protected:
     Linked<IKeyArray> keyArray;
     const RtlRecord *keyRecInfo = nullptr;
     bool createSegmentMonitorsPending;
+    virtual bool hasNewSegmentMonitors() = 0;
 
     virtual void createSegmentMonitors() = 0;
     virtual void setPartNo(bool filechanged)
@@ -2455,7 +2446,7 @@ protected:
             }
             if (allKeys->numParts())
             {
-                tlk.setown(createKeyMerger(*keyRecInfo, allKeys, 0, &logctx));
+                tlk.setown(createKeyMerger(*keyRecInfo, allKeys, 0, &logctx, hasNewSegmentMonitors()));
                 createSegmentMonitorsPending = true;
             }
             else
@@ -2468,7 +2459,7 @@ protected:
             IKeyIndex *k = kib->queryPart(lastPartNo.fileNo);
             if (filechanged)
             {
-                tlk.setown(createLocalKeyManager(*keyRecInfo, k, &logctx));
+                tlk.setown(createLocalKeyManager(*keyRecInfo, k, &logctx, hasNewSegmentMonitors()));
                 createSegmentMonitorsPending = true;
             }
             else
@@ -2523,6 +2514,8 @@ protected:
     SmartStepExtra stepExtra; // just used for flags - a little unnecessary...
     const byte *steppingRow;
 
+    virtual bool hasNewSegmentMonitors() { return indexHelper->hasNewSegmentMonitors(); }
+
     bool checkLimit(unsigned __int64 limit)
     {
         assertex(!resent);
@@ -2716,7 +2709,7 @@ public:
                 i++;
             }
             if (allKeys->numParts())
-                tlk.setown(::createKeyMerger(*keyRecInfo, allKeys, steppingOffset, &logctx));
+                tlk.setown(::createKeyMerger(*keyRecInfo, allKeys, steppingOffset, &logctx, hasNewSegmentMonitors()));
             else
                 tlk.clear();
             createSegmentMonitorsPending = true;
@@ -4038,6 +4031,8 @@ public:
         }
     }
 
+    virtual bool hasNewSegmentMonitors() { return helper->hasNewSegmentMonitors(); }
+
     ~CRoxieKeyedJoinIndexActivity()
     {
     }
@@ -4157,12 +4152,6 @@ IMessagePacker *CRoxieKeyedJoinIndexActivity::process()
             if (rootIndex)
                 rootIndex->mergeSegmentMonitors(tlk);
             tlk->finishSegmentMonitors();
-            if (logctx.queryTraceLevel() >= 20)
-            {
-                StringBuffer out;
-                printKeyedValues(out, tlk, helper->queryIndexRecordSize());
-                logctx.CTXLOG("Using filter %s", out.str());
-            }
 
             if (!resent && (atmost != (unsigned) -1) && ((atmost > preabortKeyedJoinsThreshold) || (joinFlags & JFcountmatchabortlimit) || (keepLimit != 0)))  
             {

roxie/ccd/ccdserver.cpp

@@ -21784,16 +21784,6 @@ public:
             postFilter.addFilter(*filter);
     }
 
-    virtual unsigned ordinality() const
-    {
-        throwUnexpected();
-    }
-
-    virtual IKeySegmentMonitor *item(unsigned idx) const
-    {
-        throwUnexpected();
-    }
-
     virtual void stop()
     {
         if (useRemote())
@@ -22953,7 +22943,7 @@ public:
                                     if ((indexHelper.getFlags() & TIRcountkeyedlimit) != 0)
                                     {
                                         Owned<IKeyManager> countKey;
-                                        countKey.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, this));
+                                        countKey.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, this, indexHelper.hasNewSegmentMonitors()));
                                         countKey->setLayoutTranslator(translators->queryTranslator(fileNo));
                                         createSegmentMonitors(countKey);
                                         unsigned __int64 count = countKey->checkCount(keyedLimit);
@@ -22968,11 +22958,11 @@ public:
                             }
                             if (seekGEOffset && !thisKey->isTopLevelKey())
                             {
-                                tlk.setown(createSingleKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, seekGEOffset, this));
+                                tlk.setown(createSingleKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, seekGEOffset, this, indexHelper.hasNewSegmentMonitors()));
                             }
                             else
                             {
-                                tlk.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, this));
+                                tlk.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, this, indexHelper.hasNewSegmentMonitors()));
                                 if (!thisKey->isTopLevelKey())
                                     tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                                 else
@@ -22982,7 +22972,7 @@ public:
                             if (queryTraceLevel() > 3 || ctx->queryProbeManager())
                             {
                                 StringBuffer out;
-                                printKeyedValues(out, tlk, indexHelper.queryDiskRecordSize());
+                                tlk->describeFilter(out);
                                 CTXLOG("Using filter %s", out.str());
                                 if (ctx->queryProbeManager())
                                     ctx->queryProbeManager()->setNodeProperty(this, "filter", out.str());
@@ -23237,9 +23227,9 @@ public:
             keySet.setown(createKeyIndexSet());
             keySet->addIndex(LINK(key));
             if (owner.seekGEOffset)
-                tlk.setown(createKeyMerger(owner.indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keySet, owner.seekGEOffset, &owner));
+                tlk.setown(createKeyMerger(owner.indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keySet, owner.seekGEOffset, &owner, owner.indexHelper.hasNewSegmentMonitors()));
             else
-                tlk.setown(createLocalKeyManager(owner.indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keySet->queryPart(0), &owner));
+                tlk.setown(createLocalKeyManager(owner.indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keySet->queryPart(0), &owner, owner.indexHelper.hasNewSegmentMonitors()));
             if (!key->isTopLevelKey())
                 tlk->setLayoutTranslator(trans);
             owner.indexHelper.createSegmentMonitors(tlk);
@@ -23652,7 +23642,7 @@ public:
         unsigned __int64 result = 0;
         for (unsigned i = 0; i < numParts; i++)
         {
-            Owned<IKeyManager> countTlk = createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet->queryPart(i), this);
+            Owned<IKeyManager> countTlk = createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet->queryPart(i), this, indexHelper.hasNewSegmentMonitors());
             countTlk->setLayoutTranslator(translators->queryTranslator(i));
             indexHelper.createSegmentMonitors(countTlk);
             countTlk->finishSegmentMonitors();
@@ -23684,12 +23674,12 @@ public:
             }
             if (numParts > 1 || seekGEOffset)
             {
-                tlk.setown(createKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet, seekGEOffset, this));
+                tlk.setown(createKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet, seekGEOffset, this, indexHelper.hasNewSegmentMonitors()));
                 // note that we don't set up translator because we don't support it. If that ever changes...
             }
             else
             {
-                tlk.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet->queryPart(0), this));
+                tlk.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet->queryPart(0), this, indexHelper.hasNewSegmentMonitors()));
                 tlk->setLayoutTranslator(translators->queryTranslator(0));
             }
             indexHelper.createSegmentMonitors(tlk);
@@ -23697,7 +23687,7 @@ public:
             if (queryTraceLevel() > 3 || ctx->queryProbeManager())
             {
                 StringBuffer out;
-                printKeyedValues(out, tlk, indexHelper.queryDiskRecordSize());
+                tlk->describeFilter(out);
                 CTXLOG("Using filter %s", out.str());
                 if (ctx->queryProbeManager())
                     ctx->queryProbeManager()->setNodeProperty(this, "filter", out.str());
@@ -25324,7 +25314,7 @@ public:
     CRoxieServerFullKeyedJoinHead(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, IKeyArray * _keySet, ITranslatorSet *_translators, IOutputMetaData *_indexReadMeta, IJoinProcessor *_joinHandler, bool _isLocal)
         : CRoxieServerActivity(_ctx, _factory, _probeManager),
           helper((IHThorKeyedJoinArg &)basehelper), 
-          tlk(createLocalKeyManager(helper.queryIndexRecordSize()->queryRecordAccessor(true), NULL, this)),
+          tlk(createLocalKeyManager(helper.queryIndexRecordSize()->queryRecordAccessor(true), NULL, this, helper.hasNewSegmentMonitors())),
           translators(_translators),
           keySet(_keySet),
           remote(_ctx, this, _remoteId, 0, helper, *this, true, true),
@@ -26228,7 +26218,7 @@ public:
         IOutputMetaData *_indexReadMeta, unsigned _joinFlags, bool _isSimple, bool _isLocal)
         : CRoxieServerKeyedJoinBase(_ctx, _factory, _probeManager, _remoteId, _joinFlags, false, _isSimple, _isLocal),
           indexReadMeta(_indexReadMeta),
-          tlk(createLocalKeyManager(helper.queryIndexRecordSize()->queryRecordAccessor(true), NULL, this)),
+          tlk(createLocalKeyManager(helper.queryIndexRecordSize()->queryRecordAccessor(true), NULL, this, helper.hasNewSegmentMonitors())),
           keySet(_keySet),
           translators(_translators)
     {

rtl/eclrtl/CMakeLists.txt

@@ -37,6 +37,7 @@ set (    SRCS
          rtlint.cpp 
          rtlkey.cpp 
          rtlnewkey.cpp
+         rtlnktest.cpp
          rtlqstr.cpp 
          rtlrank.cpp 
          rtlfield.cpp 

rtl/eclrtl/rtlkey.cpp

@@ -30,15 +30,10 @@ protected:
     size32_t size;
     size32_t offset;
     unsigned fieldIdx;
-    unsigned hash;
 
 public:
     IMPLEMENT_IINTERFACE;
     CKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, unsigned _size);
-    CKeySegmentMonitor(MemoryBuffer &mb)
-    {
-        mb.read(size).read(offset).read(fieldIdx).read(hash);
-    }
 
     virtual bool matchesBuffer(const void * rawRow) const override = 0;
     virtual bool matches(const RtlRow * rawRow) const override
@@ -47,7 +42,7 @@ public:
     }
 
     virtual bool increment(void *keyval) const override;
-    virtual unsigned getFieldIdx() const override { return fieldIdx; }
+    virtual unsigned queryFieldIndex() const override { return fieldIdx; }
     virtual unsigned getOffset() const override { return offset; }
     virtual unsigned getSize() const override { return size; }
     virtual bool isWild() const override { return false; }
@@ -63,30 +58,11 @@ public:
         return memcmp(lptr, rptr, size);
     }
 
-    virtual bool equivalentTo(const IKeySegmentMonitor &other) const override
-    {
-        return offset==other.getOffset() 
-            && size==other.getSize() 
-            && isSigned()==other.isSigned() 
-            && isLittleEndian()==other.isLittleEndian();
-    }
-
-    virtual unsigned queryHashCode() const override
-    {
-        return hash;
-    }
-
     virtual bool getBloomHash(hash64_t &hash) const override
     {
         return false;
     }
 
-    virtual bool setOffset(unsigned _offset) override
-    {
-        offset = _offset;
-        return true;
-    }
-
     virtual void setHigh(void *keyval) const override;
 
     virtual void copy(void * l, const void * r) const override
@@ -95,25 +71,12 @@ public:
         char *rptr = ((char *) r) + offset;
         memcpy(lptr, rptr, size);
     }
-
-    virtual MemoryBuffer &serialize(MemoryBuffer &mb) const override
-    {
-        KeySegmentMonitorSerializeType typ = serializeType();
-        assertex(typ!=KSMST_none);
-        return mb.append((byte)typ).append(size).append(offset).append(hash);
-    }
-
-    virtual KeySegmentMonitorSerializeType serializeType() const override = 0;
 };
 
 class CWildKeySegmentMonitor : public CKeySegmentMonitor
 {
 public:
     CWildKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, unsigned _size);
-    CWildKeySegmentMonitor(MemoryBuffer &mb)
-        : CKeySegmentMonitor(mb)
-    {
-    }
     virtual bool matchesBuffer(const void *keyval) const override;
     virtual int docompare(const void *,const void *) const override;
     virtual void setLow(void *keyval) const override;
@@ -122,8 +85,7 @@ public:
     virtual bool isSimple() const override { return true; }
     virtual bool isWellKeyed() const override { return false; }
     virtual bool isOptional() const override { return true; }
-    virtual IKeySegmentMonitor *clone() const override;
-    virtual KeySegmentMonitorSerializeType serializeType() const override { return KSMST_WILDKEYSEGMENTMONITOR; }
+    virtual StringBuffer &describe(StringBuffer &out, const RtlTypeInfo &type) const override { return out.append('*'); }
 };
 
 class CSetKeySegmentMonitor : public CKeySegmentMonitor
@@ -134,12 +96,6 @@ private:
 
 public:
     CSetKeySegmentMonitor(bool _optional, IStringSet *set, unsigned _fieldIdx, unsigned _offset, unsigned _size);
-    CSetKeySegmentMonitor(MemoryBuffer &mb)
-        : CKeySegmentMonitor(mb)
-    {
-        set.setown(deserializeStringSet(mb));
-        mb.read(optional);
-    }
 
 // IKeySegmentMonitor
     virtual bool increment(void *keyval) const override;
@@ -152,7 +108,6 @@ public:
     virtual bool isSimple() const override { return true; }
     virtual bool isSigned() const override { return set->isSigned(); }
     virtual bool isLittleEndian() const override { return !set->isBigEndian(); }
-    virtual IKeySegmentMonitor *clone() const override;
 
     virtual int docompare(const void * l, const void * r) const override
     {
@@ -160,15 +115,7 @@ public:
         char *rptr = ((char *) r) + offset;
         return set->memcmp(lptr, rptr, size);
     }
-
-    virtual MemoryBuffer &serialize(MemoryBuffer &mb) const override
-    {
-        CKeySegmentMonitor::serialize(mb);
-        set->serialize(mb);
-        return mb.append(optional);
-    }
-
-    virtual KeySegmentMonitorSerializeType serializeType() const override { return KSMST_SETKEYSEGMENTMONITOR; }
+    virtual StringBuffer &describe(StringBuffer &out, const RtlTypeInfo &type) const override { return out.append("[...]"); }  // MORE - could do better
 };
 
 CKeySegmentMonitor::CKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, unsigned _size)
@@ -176,9 +123,6 @@ CKeySegmentMonitor::CKeySegmentMonitor(unsigned _fieldIdx, unsigned _offset, uns
     size = _size;
     offset = _offset;
     fieldIdx = _fieldIdx;
-    hash = 123456;
-    hash = hashc((unsigned char *) &offset, sizeof(offset), hash);
-    hash = hashc((unsigned char *) &size, sizeof(size), hash);
 }
 
 bool CKeySegmentMonitor::increment(void *bufptr) const
@@ -207,11 +151,6 @@ CWildKeySegmentMonitor::CWildKeySegmentMonitor(unsigned _fieldIdx, unsigned _off
 {
 }
 
-IKeySegmentMonitor *CWildKeySegmentMonitor::clone() const
-{   
-    return new CWildKeySegmentMonitor(fieldIdx, offset, size);
-}
-
 bool CWildKeySegmentMonitor::matchesBuffer(const void *keyval) const
 { 
     return true;
@@ -238,13 +177,6 @@ CSetKeySegmentMonitor::CSetKeySegmentMonitor(bool _optional, IStringSet *_set, u
     : set(_set), CKeySegmentMonitor(_fieldIdx, _offset, _size)
 {
     optional = _optional;
-    hash =  FNV_32_HASHONE_VALUE(hash, (byte) set->isSigned());
-    hash =  FNV_32_HASHONE_VALUE(hash, (byte) !set->isBigEndian());
-}
-
-IKeySegmentMonitor *CSetKeySegmentMonitor::clone() const 
-{
-    return new CSetKeySegmentMonitor(optional, set.getLink(), fieldIdx, offset, size);
 }
 
 bool CSetKeySegmentMonitor::increment(void *bufptr) const
@@ -320,20 +252,6 @@ public:
         optional = _optional;
     }
 
-    CSingleKeySegmentMonitorBase(MemoryBuffer &mb)
-        : CKeySegmentMonitor(mb)
-    {
-        bool hasval;
-        mb.read(hasval);
-        if (hasval) {
-            val = malloc(size);
-            memcpy(val,mb.readDirect(size),size);
-        }
-        else
-            val = NULL;
-        mb.read(optional);
-    }
-
     ~CSingleKeySegmentMonitorBase()
     {
         free(val);
@@ -343,7 +261,7 @@ public:
     virtual bool increment(void *bufptr) const override
     {
         // Set to next permitted value above current
-        if (docompare(bufptr, ((char *) val)-offset) < 0)
+        if (docompare(bufptr, ((char *) val)-offset) < 0)  // NOTE - we subtract offset from val before passing because the compare will add it on again. Confusing or what...
         {
             char *ptr = ((char *) bufptr) + offset;
             memcpy(ptr, val, size);
@@ -368,23 +286,21 @@ public:
     virtual void endRange(void *bufptr) const override
     {
         // Set to last permitted value in the range that includes current (which is asserted to be valid)
-#ifdef DEBUG
-        assertex(matchesBuffer(bufptr));
-#endif
+        dbgassertex(matchesBuffer(bufptr));
     }
 
     virtual bool isWellKeyed() const override { return true; }
     virtual bool isOptional() const override { return optional; }
     virtual bool isSimple() const override { return true; }
-
-    virtual MemoryBuffer &serialize(MemoryBuffer &mb) const override
+    virtual StringBuffer &describe(StringBuffer &out, const RtlTypeInfo &type) const override
     {
-        CKeySegmentMonitor::serialize(mb);
-        if (val) 
-            mb.append((bool)true).append(size,val);
+        size32_t size;
+        rtlDataAttr text;
+        type.getString(size, text.refstr(), val);
+        if (type.isNumeric())
+            return out.append(size, text.getstr());
         else
-            mb.append((bool)false);
-        return mb.append(optional);
+            return out.appendf("'%*s'", size, text.getstr());
     }
 };
 
@@ -394,17 +310,6 @@ public:
     CSingleKeySegmentMonitor(bool _optional, const void *_val, unsigned _fieldIdx, unsigned _offset, unsigned _size)
         : CSingleKeySegmentMonitorBase(_optional, _val, _fieldIdx, _offset, _size)
     {
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 0);
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 0);
-    }
-    CSingleKeySegmentMonitor(MemoryBuffer &mb)
-        : CSingleKeySegmentMonitorBase(mb)
-    {
-    }
-
-    virtual IKeySegmentMonitor *clone() const override
-    {
-        return new CSingleKeySegmentMonitor(optional, val, fieldIdx, offset, size);
     }
 
     virtual bool isSigned() const override { return false; }
@@ -416,8 +321,6 @@ public:
         hash = rtlHash64Data(size, val, hash);
         return true;
     }
-
-    virtual KeySegmentMonitorSerializeType serializeType() const override { return KSMST_SINGLEKEYSEGMENTMONITOR; }
 };
 
 class CSingleBigSignedKeySegmentMonitor : public CSingleKeySegmentMonitorBase
@@ -426,18 +329,6 @@ public:
     CSingleBigSignedKeySegmentMonitor(bool _optional, const void *_val, unsigned _fieldIdx, unsigned _offset, unsigned _size)
         : CSingleKeySegmentMonitorBase(_optional, _val, _fieldIdx, _offset, _size)
     {
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 1);
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 0);
-    }
-
-    CSingleBigSignedKeySegmentMonitor(MemoryBuffer &mb)
-        : CSingleKeySegmentMonitorBase(mb)
-    {
-    }
-
-    virtual IKeySegmentMonitor *clone() const override
-    {
-        return new CSingleBigSignedKeySegmentMonitor(optional, val, fieldIdx, offset, size);
     }
 
     virtual int docompare(const void *l, const void *r) const override
@@ -447,7 +338,6 @@ public:
 
     virtual bool isSigned() const override { return true; }
     virtual bool isLittleEndian() const override { return false; }
-    virtual KeySegmentMonitorSerializeType serializeType() const override { return KSMST_SINGLEBIGSIGNEDKEYSEGMENTMONITOR; }
 };
 
 class CSingleLittleSignedKeySegmentMonitor : public CSingleKeySegmentMonitorBase
@@ -456,18 +346,6 @@ public:
     CSingleLittleSignedKeySegmentMonitor(bool _optional, const void *_val, unsigned _fieldIdx, unsigned _offset, unsigned _size)
         : CSingleKeySegmentMonitorBase(_optional, _val, _fieldIdx, _offset, _size)
     {
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 1);
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 1);
-    }
-
-    CSingleLittleSignedKeySegmentMonitor(MemoryBuffer &mb)
-        : CSingleKeySegmentMonitorBase(mb)
-    {
-    }
-
-    virtual IKeySegmentMonitor *clone() const override
-    {
-        return new CSingleLittleSignedKeySegmentMonitor(optional, val, fieldIdx, offset, size);
     }
 
     virtual int docompare(const void *l, const void *r) const
@@ -477,7 +355,6 @@ public:
 
     virtual bool isSigned() const override { return true; }
     virtual bool isLittleEndian() const override { return true; }
-    virtual KeySegmentMonitorSerializeType serializeType() const override { return KSMST_SINGLELITTLESIGNEDKEYSEGMENTMONITOR; }
 };
 
 class CSingleLittleKeySegmentMonitor : public CSingleKeySegmentMonitorBase
@@ -486,18 +363,6 @@ public:
     CSingleLittleKeySegmentMonitor(bool _optional, const void *_val, unsigned _fieldIdx, unsigned _offset, unsigned _size)
         : CSingleKeySegmentMonitorBase(_optional, _val, _fieldIdx, _offset, _size)
     {
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 0);
-        hash = FNV_32_HASHONE_VALUE(hash, (byte) 1);
-    }
-
-    CSingleLittleKeySegmentMonitor(MemoryBuffer &mb)
-        : CSingleKeySegmentMonitorBase(mb)
-    {
-    }
-
-    virtual IKeySegmentMonitor *clone() const override
-    {
-        return new CSingleLittleKeySegmentMonitor(optional, val, fieldIdx, offset, size);
     }
 
     virtual int docompare(const void *l, const void *r) const override
@@ -507,7 +372,6 @@ public:
 
     virtual bool isSigned() const override { return false; }
     virtual bool isLittleEndian() const override { return true; }
-    virtual KeySegmentMonitorSerializeType serializeType() const override { return KSMST_CSINGLELITTLEKEYSEGMENTMONITOR; }
 };
 
 ECLRTL_API IStringSet *createRtlStringSet(size32_t size)
@@ -635,37 +499,6 @@ ECLRTL_API IKeySegmentMonitor *createDummyKeySegmentMonitor(unsigned _fieldIdx,
             return new CSingleKeySegmentMonitor(false, NULL, _fieldIdx, _offset, _size);
         }
 
-ECLRTL_API IKeySegmentMonitor *deserializeKeySegmentMonitor(MemoryBuffer &mb)
-{
-    byte typ;
-    mb.read(typ);
-    switch ((KeySegmentMonitorSerializeType)typ) {
-        case KSMST_WILDKEYSEGMENTMONITOR:
-            return new CWildKeySegmentMonitor(mb);
-        case KSMST_SETKEYSEGMENTMONITOR:
-            return new CSetKeySegmentMonitor(mb);
-        case KSMST_SINGLEKEYSEGMENTMONITOR:
-            return new CSingleKeySegmentMonitor(mb);
-        case KSMST_SINGLEBIGSIGNEDKEYSEGMENTMONITOR:
-            return new CSingleBigSignedKeySegmentMonitor(mb);
-        case KSMST_SINGLELITTLESIGNEDKEYSEGMENTMONITOR:
-            return new CSingleLittleSignedKeySegmentMonitor(mb);
-        case KSMST_CSINGLELITTLEKEYSEGMENTMONITOR:
-            return new CSingleLittleKeySegmentMonitor(mb);
-    }
-    return NULL; // up to caller to check
-}
-
-enum StringSetSerializeType
-{
-    SSST_none,
-    SSST_BIGUNSIGNEDSTRINGSET,
-    SSST_BIGSIGNEDSTRINGSET,
-    SSST_LITTLEUNSIGNEDSTRINGSET,
-    SSST_LITTLESIGNEDSTRINGSET,
-    SSST_max
-};
-
 ECLRTL_API int memcmpbigsigned(const void *l, const void *r, unsigned size)
 {
     signed int diff = ((signed char *) l)[0]-((signed char *) r)[0];
@@ -732,7 +565,6 @@ protected:
 public:
     IMPLEMENT_IINTERFACE;
     CStringSet(size32_t size);
-    CStringSet(MemoryBuffer &mb);
 
 // IStringSet
     virtual void addRange(const void *loval, const void *hival);
@@ -776,19 +608,6 @@ public:
         }
         return ret;
     }
-
-    virtual MemoryBuffer &serialize(MemoryBuffer &mb) const
-    {
-        StringSetSerializeType typ = serializeType();
-        assertex(typ!=SSST_none);
-        mb.append((byte)typ).append(size).append(transitions.ordinality());
-        ForEachItemIn(i,transitions) {
-            transitions.item(i).serialize(size,mb);
-        }
-        return mb;
-    }
-
-    virtual StringSetSerializeType serializeType() const = 0;
 };
 
 class CBigUnsignedStringSet : public CStringSet
@@ -880,17 +699,8 @@ protected:
     virtual bool isSigned() const { return false; }
     virtual bool isBigEndian() const { return true; }
 
-
-    virtual StringSetSerializeType serializeType() const
-    {
-        return SSST_BIGUNSIGNEDSTRINGSET;
-    }
-
 public:
     CBigUnsignedStringSet(unsigned size) : CStringSet(size) {}
-    CBigUnsignedStringSet(MemoryBuffer &mb) : CStringSet(mb) {}
-
-
 };
 
 class CBigSignedStringSet : public CBigUnsignedStringSet
@@ -949,13 +759,8 @@ protected:
     virtual bool isSigned() const { return true; }
     virtual bool isBigEndian() const { return true; }
 
-    virtual StringSetSerializeType serializeType() const
-    {
-        return SSST_BIGSIGNEDSTRINGSET;
-    }
 public:
     CBigSignedStringSet(unsigned size) : CBigUnsignedStringSet(size) {}
-    CBigSignedStringSet(MemoryBuffer &mb) : CBigUnsignedStringSet(mb) {}
 };
 
 class CLittleUnsignedStringSet : public CStringSet
@@ -1050,15 +855,8 @@ protected:
 
     virtual bool isSigned() const { return false; }
     virtual bool isBigEndian() const { return false; }
-
-    virtual StringSetSerializeType serializeType() const
-    {
-        return SSST_LITTLEUNSIGNEDSTRINGSET;
-    }
-
 public:
     CLittleUnsignedStringSet(unsigned size) : CStringSet(size) {}
-    CLittleUnsignedStringSet(MemoryBuffer &mb) : CStringSet(mb) {}
 };
 
 class CLittleSignedStringSet : public CLittleUnsignedStringSet
@@ -1118,15 +916,8 @@ protected:
 
     virtual bool isSigned() const { return true; }
     virtual bool isBigEndian() const { return false; }
-
-    virtual StringSetSerializeType serializeType() const
-    {
-        return SSST_LITTLESIGNEDSTRINGSET;
-    }
-
 public:
     CLittleSignedStringSet(unsigned size) : CLittleUnsignedStringSet(size) {}
-    CLittleSignedStringSet(MemoryBuffer &mb) : CLittleUnsignedStringSet(mb) {}
 };
 
 
@@ -1144,50 +935,21 @@ public:
         val = _val;
         state = _state;
     }
-
-    CTransition(MemoryBuffer &mb,size32_t size)
-    {
-        mb.read(state);
-        val = malloc(size);
-        memcpy((void *)val,mb.readDirect(size),size);
-    }
-
     ~CTransition() { free((void *) val); }
 
 // ITransition
 
-    bool getState() const { return state; }
-    const void *getValue() const { return val; }
-
-    MemoryBuffer &serialize(size32_t size, MemoryBuffer &mb) const
-    {
-        mb.append(state);
-        memcpy(mb.reserve(size),val,size);
-        return mb;
-    }
-
-    bool canSerialize() const  { return true; }
-
+    virtual bool getState() const override { return state; }
+    virtual const void *getValue() const override { return val; }
 };
 
 //======================================================================================
 
-
 CStringSet::CStringSet(size32_t _size)
 {
     size = _size;
 }
 
-CStringSet::CStringSet(MemoryBuffer &mb)
-{
-    mb.read(size);
-    unsigned n;
-    mb.read(n);
-    while(n--) 
-        transitions.append(*new CTransition(mb,size));
-}
-
-
 void CStringSet::reset()
 {
     transitions.kill();
@@ -1625,23 +1387,6 @@ IStringSet *createStringSet(size32_t size, bool bigEndian, bool isSigned)
     }
 }
 
-ECLRTL_API IStringSet *deserializeStringSet(MemoryBuffer &mb)
-{
-    byte typ;
-    mb.read(typ);
-    switch((StringSetSerializeType)typ) {
-    case SSST_BIGUNSIGNEDSTRINGSET:
-        return new CBigUnsignedStringSet(mb);
-    case SSST_BIGSIGNEDSTRINGSET:
-        return new CBigSignedStringSet(mb);
-    case SSST_LITTLEUNSIGNEDSTRINGSET:
-        return new CLittleUnsignedStringSet(mb);
-    case SSST_LITTLESIGNEDSTRINGSET:
-        return new CLittleSignedStringSet(mb);
-    }
-    return NULL; // up to caller to check
-};
-
 //---------------------------------------------------------------------------------------------------------------------
 
 class LegacySetCreator : implements ISetCreator

rtl/eclrtl/rtlkey.hpp

@@ -19,23 +19,10 @@
 #define RTLKEY_INCL
 #include "eclrtl.hpp"
 
-enum KeySegmentMonitorSerializeType
-{
-    KSMST_none,                  // can't serialize
-    KSMST_WILDKEYSEGMENTMONITOR,
-    KSMST_SETKEYSEGMENTMONITOR,
-    KSMST_SINGLEKEYSEGMENTMONITOR,
-    KSMST_SINGLEBIGSIGNEDKEYSEGMENTMONITOR,
-    KSMST_SINGLELITTLESIGNEDKEYSEGMENTMONITOR,
-    KSMST_CSINGLELITTLEKEYSEGMENTMONITOR,
-    KSMST_max
-};
-
 interface ITransition : extends IInterface
 {
     virtual bool getState() const = 0;
     virtual const void *getValue() const = 0;
-    virtual MemoryBuffer &serialize(size32_t size, MemoryBuffer &buffer) const = 0;
 };
 
 interface IStringSet : public IInterface
@@ -64,12 +51,10 @@ interface IStringSet : public IInterface
     virtual int memcmp(const void *val1, const void *val2, size32_t size) const = 0; 
     virtual bool decrement(void *val) const = 0;
     virtual bool increment(void *val) const = 0;
-    virtual MemoryBuffer &serialize(MemoryBuffer &buffer) const = 0;
 };
 
 ECLRTL_API IStringSet *createStringSet(size32_t size);
 ECLRTL_API IStringSet *createStringSet(size32_t size, bool bigEndian, bool isSigned);
-ECLRTL_API IStringSet *deserializeStringSet(MemoryBuffer &mb);
 
 ECLRTL_API int memcmpbigsigned(const void *l, const void *r, unsigned size);
 ECLRTL_API int memcmplittleunsigned(const void *l, const void *r, unsigned size);
@@ -78,7 +63,17 @@ ECLRTL_API int memcmplittlesigned(const void *l, const void *r, unsigned size);
 class RtlRow;
 class RtlRecord;
 
-interface IKeySegmentMonitor : public IInterface
+// Common base interface between new and old-style field filters
+interface IIndexFilter : public IInterface
+{
+    virtual bool getBloomHash(hash64_t &hash) const = 0;
+    virtual unsigned queryFieldIndex() const = 0;
+
+    virtual bool isWild() const = 0;
+    virtual bool isEmpty() const = 0;
+};
+
+interface IKeySegmentMonitor : public IIndexFilter
 {
 public:
     virtual bool increment(void * expandedRow) const = 0;
@@ -86,32 +81,24 @@ public:
     virtual void setHigh(void * expandedRow) const = 0;
     virtual void endRange(void * expandedRow) const = 0;
 
-    virtual bool matchesBuffer(const void * expandedRow) const = 0;
     virtual bool matches(const RtlRow * rawRow) const = 0;
+    virtual bool matchesBuffer(const void * expandedRow) const = 0;
 
-    virtual bool isWild() const = 0;
-    virtual unsigned getFieldIdx() const = 0;
     virtual unsigned getOffset() const = 0;
     virtual unsigned getSize() const = 0;
-    virtual bool isEmpty() const = 0;
-    virtual bool equivalentTo(const IKeySegmentMonitor &other) const = 0;
     virtual bool isSigned() const = 0;
     virtual bool isLittleEndian() const = 0;
 
     virtual int docompare(const void * expandedLeft, const void * rawRight) const = 0;
-    virtual unsigned queryHashCode() const = 0;
     virtual bool isWellKeyed() const = 0;
     virtual bool isOptional() const = 0;
 
     virtual bool isSimple() const = 0;
     virtual void copy(void *expandedRow, const void *rawRow) const = 0;
-    virtual MemoryBuffer &serialize(MemoryBuffer &mb) const = 0;
-    virtual KeySegmentMonitorSerializeType serializeType() const = 0;
-    virtual IKeySegmentMonitor *clone() const = 0;
     virtual unsigned numFieldsRequired() const = 0;
-    virtual bool getBloomHash(hash64_t &hash) const = 0;
 
-    virtual bool setOffset(unsigned _offset) = 0;  // Used by old record layout translator - to be removed at some point
+    // Describe for the purposes of tracing/debugging
+    virtual StringBuffer &describe(StringBuffer &out, const RtlTypeInfo &type) const = 0;
 };
 
 interface IBlobProvider
@@ -136,8 +123,6 @@ interface IIndexReadContext
 {
 public:
     virtual void append(IKeySegmentMonitor *segment) = 0;
-    virtual unsigned ordinality() const = 0;
-    virtual IKeySegmentMonitor *item(unsigned idx) const = 0;
     virtual void append(FFoption option, const IFieldFilter * filter) = 0;
 };
 
@@ -238,8 +223,19 @@ interface IValueSet : public IInterface
     virtual bool matches(const byte * field, unsigned range) const = 0;
 
     virtual const RtlTypeInfo & queryType() const = 0;
+
+    // Is this a single-valued set?
+    virtual const void *querySingleValue() const = 0;
+
+    // jhtree usage
+    virtual void setLow(void *buffer, size32_t offset, const RtlTypeInfo &parentType) const = 0;
+    virtual bool incrementKey(void *buffer, size32_t offset, const RtlTypeInfo &parentType) const = 0;
+    virtual void endRange(void *buffer, size32_t offset, const RtlTypeInfo &parentType) const = 0;
+    virtual void setHigh(void *buffer, size32_t offset, const RtlTypeInfo &parentType) const = 0;
+
 };
 extern ECLRTL_API IValueSet * createValueSet(const RtlTypeInfo & type);
+extern ECLRTL_API IValueSet * createValueSet(const RtlTypeInfo & _type, MemoryBuffer & in);
 
 interface ISetCreator
 {
@@ -284,12 +280,11 @@ extern ECLRTL_API void deserializeSet(IValueSet & set, const char * filter);
  *
  * Example implementations include single value, sets of ranges, regex or wildcard
  */
-interface IFieldFilter : public IInterface
+interface IFieldFilter : public IIndexFilter
 {
 public:
 //Simple row matching
     virtual bool matches(const RtlRow & row) const = 0;
-    virtual unsigned queryFieldIndex() const = 0;
     virtual const RtlTypeInfo & queryType() const = 0;
     virtual int compareRow(const RtlRow & left, const RtlRow & right) const = 0;
     virtual int compareLowest(const RtlRow & left, unsigned range) const = 0;
@@ -304,6 +299,15 @@ public:
     virtual int findForwardMatchRange(const RtlRow & row, unsigned & matchRange) const = 0;
     virtual unsigned queryScore() const = 0;
     virtual IFieldFilter *remap(unsigned newFieldIndex) const = 0;
+
+    // For use with jhtree
+    virtual void setLow(void *buffer, size32_t offset) const = 0;
+    virtual bool incrementKey(void *buffer, size32_t offset) const = 0;
+    virtual void endRange(void *buffer, size32_t offset) const = 0;
+    virtual void setHigh(void *buffer, size32_t offset) const = 0;
+
+    // Human-readable description for tracing/debugging
+    virtual StringBuffer &describe(StringBuffer &out) const = 0;
 };
 
 //More types of IFieldFilter to come later
@@ -316,7 +320,9 @@ extern ECLRTL_API IFieldFilter * createSubStringFieldFilter(unsigned fieldId, si
 
 extern ECLRTL_API IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type, const char * src);
 extern ECLRTL_API IFieldFilter * deserializeFieldFilter(const RtlRecord & record, const char * src);
+extern ECLRTL_API IFieldFilter * deserializeFieldFilter(const RtlRecord & searchRecord, MemoryBuffer & in);
 extern ECLRTL_API IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type, MemoryBuffer & in);
+
 extern ECLRTL_API void readFieldFromFieldFilter(StringBuffer & fieldText, const char * & src);
 
 

rtl/eclrtl/rtlnewkey.hpp

@@ -22,6 +22,31 @@
 #include "rtlkey.hpp"
 #include "rtlrecord.hpp"
 
+/*
+ * Common functionality required by anything wanting to act as a filter for an index
+ */
+
+interface IIndexFilterList : public IInterface, public IIndexReadContext
+{
+    virtual const IIndexFilter *item(unsigned idx) const = 0;
+    virtual void setLow(unsigned segno, void *keyBuffer) const = 0;
+    virtual unsigned setLowAfter(size32_t offset, void *keyBuffer) const = 0;
+    virtual bool incrementKey(unsigned segno, void *keyBuffer) const = 0;
+    virtual void endRange(unsigned segno, void *keyBuffer) const = 0;
+    virtual unsigned lastRealSeg() const = 0;
+    virtual unsigned lastFullSeg() const = 0;
+    virtual unsigned numFilterFields() const = 0;
+    virtual IIndexFilterList *fixSortSegs(const char *fixedVals, unsigned sortFieldOffset) const = 0;
+    virtual void reset() = 0;
+    virtual void checkSize(size32_t keyedSize, char const * keyname) const = 0;
+    virtual void recalculateCache() = 0;
+    virtual void finish(size32_t keyedSize) = 0;
+    virtual void describe(StringBuffer &out) const = 0;
+    virtual bool matchesBuffer(const void *keyBuffer, unsigned lastSeg, unsigned &matchSeg) const = 0;
+    virtual unsigned getFieldOffset(unsigned idx) const = 0;
+    virtual bool canMatch() const = 0;
+};
+
 BITMASK_ENUM(TransitionMask);
 
 /*

rtl/eclrtl/rtlrecord.cpp

@@ -631,6 +631,12 @@ RtlRow::RtlRow(const RtlRecord & _info, const void * optRow, unsigned numOffsets
     setRow(optRow);
 }
 
+size_t RtlRow::noVariableOffsets [1] = {0};
+
+RtlRow::RtlRow(const RtlRecord & _info) : info(_info), variableOffsets(noVariableOffsets)
+{
+}
+
 __int64 RtlRow::getInt(unsigned field) const
 {
     const byte * self = reinterpret_cast<const byte *>(row);
@@ -730,6 +736,14 @@ void RtlRow::lazyCalcOffsets(unsigned _numFieldsUsed) const
     }
 }
 
+RtlFixedRow::RtlFixedRow(const RtlRecord & _info, const void *_row, unsigned _numFieldsUsed)
+: RtlRow(_info)
+{
+    row = (const byte *) _row;
+    numFieldsUsed = _numFieldsUsed;
+    dbgassertex(info.isFixedOffset(numFieldsUsed));
+}
+
 RtlDynRow::RtlDynRow(const RtlRecord & _info, const void * optRow) : RtlRow(_info, optRow, _info.getNumVarFields()+1, new size_t[_info.getNumVarFields()+1])
 {
 }

rtl/eclrtl/rtlrecord.hpp

@@ -294,10 +294,14 @@ public:
     }
     explicit operator bool() { return row != nullptr; }
 protected:
+    RtlRow(const RtlRecord & _info);  // for use by fixed-only case
+
     const RtlRecord & info;
     const byte * row;
     mutable unsigned numFieldsUsed = 0;
     size_t * variableOffsets;       // [0 + 1 entry for each variable size field ]
+    static size_t noVariableOffsets [1];  // Used when we are only interested in fixed offsets
+
 };
 
 struct ECLRTL_API RtlDynRow : public RtlRow
@@ -307,6 +311,15 @@ public:
     ~RtlDynRow();
 };
 
+// Special case for where we only want to access fields that are at fixed offsets - i.e. jhtree
+// Note that the RecInfo passed in may have variable fields, but we should not (and must not) try to resolve them
+
+struct ECLRTL_API RtlFixedRow : public RtlRow
+{
+public:
+    RtlFixedRow(const RtlRecord & _info, const void *_row, unsigned numFieldsUsed);
+};
+
 //The following template class is used from the generated code to avoid allocating the offset array
 template <unsigned NUM_VARIABLE_FIELDS>
 struct ECLRTL_API RtlStaticRow : RtlRow

rtl/include/eclhelper.hpp

@@ -1136,6 +1136,7 @@ enum
     TIRunfilteredtransform = 0x00004000,
     TIRorderedmerge     = 0x00008000,
     TIRunordered        = 0x00010000,
+    TIRnewfilters       = 0x00020000,               // Uses new style field filters
 };
 
 //flags for thor index write
@@ -1665,6 +1666,7 @@ enum {
     JFsmart                      = 0x10000000,
     JFunstable                   = 0x20000000, // can sorts be unstable?
     JFnevermatchself             = 0x40000000, // for a self join can a record match itself
+    JFnewfilters                 = 0x80000000, // using FieldFilters not segmonitors
 };
 
 // FetchFlags
@@ -1779,6 +1781,9 @@ struct IHThorKeyedJoinBaseArg : public IHThorArg
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned __int64 keyedFpos, unsigned counter) { return 0; }
 //Denormalize group:
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned _numRows, const void * * _rows) { return 0; }
+
+    inline bool hasNewSegmentMonitors()                     { return (getJoinFlags() & JFnewfilters) != 0; }
+
 };
 
 struct IHThorKeyedJoinArg : public IHThorKeyedJoinBaseArg, public IHThorFetchContext
@@ -1812,6 +1817,7 @@ enum
 {
     KDFvarindexfilename     = 0x00000001,
     KDFdynamicindexfilename = 0x00000002,
+    KDFnewfilters           = 0x00000004,
 };
 
 struct IHThorKeyedDistributeArg : public IHThorArg
@@ -1824,6 +1830,7 @@ struct IHThorKeyedDistributeArg : public IHThorArg
     virtual ICompare * queryCompareRowKey() = 0;
     virtual unsigned getFormatCrc() = 0;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) = 0;
+    inline bool hasNewSegmentMonitors() { return (getFlags() & KDFnewfilters) != 0; }
 };
 
 
@@ -2333,6 +2340,7 @@ struct IHThorIndexReadBaseArg : extends IHThorCompoundBaseArg
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) = 0;
 
     inline bool hasSegmentMonitors()                        { return (getFlags() & TIRnofilter) == 0; }
+    inline bool hasNewSegmentMonitors()                     { return (getFlags() & TIRnewfilters) != 0; }
     virtual IHThorSteppedSourceExtra *querySteppingExtra()  { return NULL; }
 };
 

system/jhtree/bloom.cpp

@@ -89,23 +89,23 @@ int IndexBloomFilter::compare(CInterface *const *_a, CInterface *const *_b)
     return a->fields - b->fields;
 }
 
-bool IndexBloomFilter::reject(const SegMonitorList &segs) const
+bool IndexBloomFilter::reject(const IIndexFilterList &filters) const
 {
     hash64_t hashval = HASH64_INIT;
-    return getBloomHash(fields, segs, hashval) && !test(hashval);
+    return getBloomHash(fields, filters, hashval) && !test(hashval);
 }
 
-extern bool getBloomHash(__int64 fields, const SegMonitorList &segs, hash64_t &hashval)
+extern bool getBloomHash(__int64 fields, const IIndexFilterList &filters, hash64_t &hashval)
 {
     while (fields)
     {
         unsigned f = ffsll(fields)-1;    // extract lowest 1 bit
         fields &= ~ (((__uint64) 1)<<f); // and clear it
-        IKeySegmentMonitor *seg = segs.item(f);
-        if (seg)
+        const IIndexFilter *filter = filters.item(f);
+        if (filter)
         {
-            assertex(seg->getFieldIdx() == f);
-            if (!seg->getBloomHash(hashval))
+            assertex(filter->queryFieldIndex() == f);
+            if (!filter->getBloomHash(hashval))
                 return false;
         }
     }
@@ -117,7 +117,6 @@ class RowHasher : public CInterfaceOf<IRowHasher>
 public:
     RowHasher(const RtlRecord &_recInfo, __uint64 _fields);
     virtual hash64_t hash(const byte *row) const override;
-    virtual bool isExact(const SegMonitorList &segs) const override;
     virtual __uint64 queryFields() const override { return fields; }
 private:
     const RtlRecord &recInfo;
@@ -152,20 +151,6 @@ hash64_t RowHasher::hash(const byte *row) const
     return hashval;
 }
 
-bool RowHasher::isExact(const SegMonitorList &segs) const
-{
-    auto lfields = fields;
-    // This will need reworking if/when non-fixed-size fields are supported (should actually become easier)
-    while (lfields)
-    {
-        unsigned f = ffsll(lfields)-1;    // extract lowest 1 bit
-        lfields &= ~ (((__uint64) 1)<<f); // and clear it
-        if (!segs.isExact(recInfo.queryType(f)->getMinSize(), recInfo.getFixedOffset(f)))
-            return false;
-    }
-    return true;
-}
-
 SimpleRowHasher::SimpleRowHasher(const RtlRecord &_recInfo, __uint64 _fields, unsigned _offset, unsigned _length)
 : RowHasher(_recInfo, _fields), offset(_offset), length(_length)
 {

system/jhtree/bloom.hpp

@@ -113,7 +113,7 @@ public:
      */
     IndexBloomFilter(unsigned numHashes, unsigned tableSize, byte *table, __uint64 fields);
     inline __int64 queryFields() const { return fields; }
-    bool reject(const SegMonitorList &segs) const;
+    bool reject(const IIndexFilterList &filters) const;
     static int compare(CInterface *const *a, CInterface *const *b);
 private:
     const __uint64 fields;
@@ -160,7 +160,6 @@ extern jhtree_decl IBloomBuilder *createBloomBuilder(const IBloomBuilderInfo &_h
 interface IRowHasher : public IInterface
 {
     virtual hash64_t hash(const byte *row) const = 0;
-    virtual bool isExact(const SegMonitorList &segs) const = 0;
     virtual __uint64 queryFields() const = 0;
 };
 
@@ -175,11 +174,11 @@ extern jhtree_decl IRowHasher * createRowHasher(const RtlRecord &recInfo, __uint
 /**
  * Retrieve bloom/partition hash corresponding to a supplied filter condition
  * @param fields   Bitmap containing field numbers
- * @param segs     Filter to be checked
+ * @param filter   Filter to be checked
  * @param hash     Initial hash value, updated to reflect supplied fields
  * return          true if the filter is suitable for bloom filtering/partitioning via returned hash value
  */
-extern jhtree_decl bool getBloomHash(__int64 fields, const SegMonitorList &segs, hash64_t &hashval);
+extern jhtree_decl bool getBloomHash(__int64 fields, const IIndexFilterList &filter, hash64_t &hashval);
 
 
 #endif

system/jhtree/jhtree.cpp

@@ -88,21 +88,21 @@ MODULE_EXIT()
 
 //#define DUMP_NODES
 
-SegMonitorList::SegMonitorList(const RtlRecord &_recInfo, bool _needWild) : recInfo(_recInfo), needWild(_needWild)
+SegMonitorList::SegMonitorList(const RtlRecord &_recInfo) : recInfo(_recInfo)
 {
     keySegCount = recInfo.getNumKeyedFields();
     reset();
 }
 
 SegMonitorList::SegMonitorList(const SegMonitorList &from, const char *fixedVals, unsigned sortFieldOffset)
-: recInfo(from.recInfo), needWild(from.needWild), keySegCount(from.keySegCount)
+: recInfo(from.recInfo), keySegCount(from.keySegCount)
 {
     ForEachItemIn(idx, from.segMonitors)
     {
         IKeySegmentMonitor &seg = from.segMonitors.item(idx);
         unsigned offset = seg.getOffset();
         if (offset < sortFieldOffset)
-            segMonitors.append(*createSingleKeySegmentMonitor(false, seg.getFieldIdx(), offset, seg.getSize(), fixedVals+offset));
+            segMonitors.append(*createSingleKeySegmentMonitor(false, seg.queryFieldIndex(), offset, seg.getSize(), fixedVals+offset));
         else
             segMonitors.append(OLINK(seg));
     }
@@ -110,12 +110,42 @@ SegMonitorList::SegMonitorList(const SegMonitorList &from, const char *fixedVals
     modified = false;
 }
 
-unsigned SegMonitorList::ordinality() const
+void SegMonitorList::describe(StringBuffer &out) const
 {
-    return segMonitors.length();
+    for (unsigned idx=0; idx < lastRealSeg(); idx++)
+    {
+        auto &filter = segMonitors.item(idx);
+        if (idx)
+            out.append(',');
+        out.appendf("%s=", recInfo.queryName(idx));
+        filter.describe(out, *recInfo.queryType(idx));
+    }
+}
+
+bool SegMonitorList::matchesBuffer(const void *buffer, unsigned lastSeg, unsigned &matchSeg) const
+{
+    if (segMonitors.length())
+    {
+        for (; matchSeg <= lastSeg; matchSeg++)
+        {
+            if (!segMonitors.item(matchSeg).matchesBuffer(buffer))
+                return false;
+        }
+    }
+    return true;
 }
 
-IKeySegmentMonitor *SegMonitorList::item(unsigned idx) const
+bool SegMonitorList::canMatch() const
+{
+    ForEachItemIn(idx, segMonitors)
+    {
+        if (segMonitors.item(idx).isEmpty())
+            return false;
+    }
+    return true;
+}
+
+IIndexFilter *SegMonitorList::item(unsigned idx) const
 {
     return &segMonitors.item(idx);
 }
@@ -132,24 +162,7 @@ size32_t SegMonitorList::getSize() const
         return 0;
 }
 
-bool SegMonitorList::isExact(unsigned len, unsigned start) const
-{
-    ForEachItemIn(idx, segMonitors)
-    {
-        IKeySegmentMonitor &seg = segMonitors.item(idx);
-        unsigned o = seg.getOffset();
-        unsigned s = seg.getSize();
-        if (o+s <= start)
-            continue;
-        if (o >= start+len)
-            return true;
-        if (!seg.isWellKeyed())
-            return false;
-    }
-    return true;
-}
-
-void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname)
+void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname) const
 {
     size32_t segSize = getSize();
     if (segSize != keyedSize)
@@ -284,32 +297,11 @@ void SegMonitorList::reset()
     modified = true;
 }
 
-void SegMonitorList::swapWith(SegMonitorList &other)
-{
-    reset();
-    other.segMonitors.swapWith(segMonitors);
-}
-
-void SegMonitorList::deserialize(MemoryBuffer &mb)
-{
-    unsigned num;
-    mb.read(num);
-    while (num--)
-        append(deserializeKeySegmentMonitor(mb));
-}
-
-void SegMonitorList::serialize(MemoryBuffer &mb) const
-{
-    mb.append((unsigned) ordinality());
-    ForEachItemIn(idx, segMonitors)
-        segMonitors.item(idx).serialize(mb);
-}
-
 // interface IIndexReadContext
 void SegMonitorList::append(IKeySegmentMonitor *segment)
 {
     modified = true;
-    unsigned fieldIdx = segment->getFieldIdx();
+    unsigned fieldIdx = segment->queryFieldIndex();
     unsigned offset = segment->getOffset();
     unsigned size = segment->getSize();
     while (segMonitors.length() < fieldIdx)
@@ -324,35 +316,23 @@ void SegMonitorList::append(IKeySegmentMonitor *segment)
 
 void SegMonitorList::append(FFoption option, const IFieldFilter * filter)
 {
-    UNIMPLEMENTED;
-}
-
-bool SegMonitorList::matched(void *keyBuffer, unsigned &lastMatch) const
-{
-    lastMatch = 0;
-    for (; lastMatch < segMonitors.length(); lastMatch++)
-    {
-        if (!segMonitors.item(lastMatch).matchesBuffer(keyBuffer))
-            return false;
-    }
-    return true;
+    throwUnexpected();
 }
 
-
 ///
 static UnexpectedVirtualFieldCallback unexpectedFieldCallback;
 class jhtree_decl CKeyLevelManager : implements IKeyManager, public CInterface
 {
 protected:
     KeyStatsCollector stats;
-    SegMonitorList segs;
+    Owned <IIndexFilterList> filter;
     IKeyCursor *keyCursor;
     ConstPointerArray activeBlobs;
     __uint64 partitionFieldMask = 0;
     unsigned indexParts = 0;
     unsigned keyedSize;     // size of non-payload part of key
-    unsigned numsegs;
     bool started = false;
+    bool newFilters = false;
 
     Owned<const IDynamicTransform> layoutTrans;
     MemoryBuffer buf;  // used when translating
@@ -360,9 +340,12 @@ protected:
 public:
     IMPLEMENT_IINTERFACE;
 
-    CKeyLevelManager(const RtlRecord &_recInfo, IKeyIndex * _key, IContextLogger *_ctx) : stats(_ctx), segs(_recInfo, true)
+    CKeyLevelManager(const RtlRecord &_recInfo, IKeyIndex * _key, IContextLogger *_ctx, bool _newFilters) : stats(_ctx), newFilters(_newFilters)
     {
-        numsegs = 0;
+        if (newFilters)
+            filter.setown(new IndexRowFilter(_recInfo));
+        else
+            filter.setown(new SegMonitorList(_recInfo));
         keyCursor = NULL;
         keyedSize = 0;
         setKey(_key);
@@ -402,7 +385,7 @@ public:
         {
             assertex(_key->numParts()==1);
             IKeyIndex *ki = _key->queryPart(0);
-            keyCursor = ki->getCursor(&segs);
+            keyCursor = ki->getCursor(filter);
             if (keyedSize)
                 assertex(keyedSize == ki->keyedSize());
             else
@@ -417,7 +400,7 @@ public:
         if (partitionFieldMask)
         {
             hash64_t hash = HASH64_INIT;
-            if (getBloomHash(partitionFieldMask, segs, hash))
+            if (getBloomHash(partitionFieldMask, *filter, hash))
                 return (((unsigned) hash) % indexParts) + 1;  // NOTE - the Hash distribute function that distributes the index when building will truncate to 32-bits before taking modulus - so we must too!
         }
         return 0;
@@ -435,8 +418,7 @@ public:
             if (!started)
             {
                 started = true;
-                numsegs = segs.ordinality();
-                segs.checkSize(keyedSize, keyCursor->queryName());
+                filter->checkSize(keyedSize, keyCursor->queryName());
             }
             if (!crappyHack)
             {
@@ -447,30 +429,21 @@ public:
 
     virtual void releaseSegmentMonitors()
     {
-        segs.reset();
+        filter->reset();
         started = false;
     }
 
     virtual void append(IKeySegmentMonitor *segment) 
     { 
-        assertex(!started);
-        segs.append(segment);
-    }
-
-
-    virtual void append(FFoption option, const IFieldFilter * filter)
-    {
-        UNIMPLEMENTED;
+        assertex(!newFilters && !started);
+        filter->append(segment);
     }
 
-    virtual unsigned ordinality() const 
-    {
-        return segs.ordinality();
-    }
 
-    virtual IKeySegmentMonitor *item(unsigned idx) const
+    virtual void append(FFoption option, const IFieldFilter * fieldFilter)
     {
-        return segs.item(idx);
+        assertex(newFilters && !started);
+        filter->append(option, fieldFilter);
     }
 
     inline const byte *queryKeyBuffer()
@@ -567,20 +540,16 @@ public:
         layoutTrans.set(trans);
     }
 
-    virtual void setSegmentMonitors(SegMonitorList &segmentMonitors) override
+    virtual void finishSegmentMonitors()
     {
-        segs.swapWith(segmentMonitors);
+        filter->finish(keyedSize);
     }
 
-    virtual void deserializeSegmentMonitors(MemoryBuffer &mb) override
+    virtual void describeFilter(StringBuffer &out) const override
     {
-        segs.deserialize(mb);
+        filter->describe(out);
     }
 
-    virtual void finishSegmentMonitors()
-    {
-        segs.finish(keyedSize);
-    }
 };
 
 
@@ -1158,9 +1127,9 @@ unsigned CKeyIndex::numPartitions()
 }
 
 
-IKeyCursor *CKeyIndex::getCursor(const SegMonitorList *segs)
+IKeyCursor *CKeyIndex::getCursor(const IIndexFilterList *filter)
 {
-    return new CKeyCursor(*this, segs);
+    return new CKeyCursor(*this, filter);
 }
 
 CJHTreeNode *CKeyIndex::getNode(offset_t offset, IContextLogger *ctx) 
@@ -1302,7 +1271,7 @@ void CKeyIndex::loadBloomFilters()
     bloomFilters.sort(IndexBloomFilter::compare);
 }
 
-bool CKeyIndex::bloomFilterReject(const SegMonitorList &segs) const
+bool CKeyIndex::bloomFilterReject(const IIndexFilterList &segs) const
 {
     ForEachItemIn(idx, bloomFilters)
     {
@@ -1407,15 +1376,15 @@ void KeyStatsCollector::reset()
     nullskips = 0;
 }
 
-CKeyCursor::CKeyCursor(CKeyIndex &_key, const SegMonitorList *_segs)
-    : key(OLINK(_key)), segs(_segs)
+CKeyCursor::CKeyCursor(CKeyIndex &_key, const IIndexFilterList *_filter)
+    : key(OLINK(_key)), filter(_filter)
 {
     nodeKey = 0;
     keyBuffer = (char *) malloc(key.keySize());  // MORE - keyedSize would do eventually
 }
 
 CKeyCursor::CKeyCursor(const CKeyCursor &from)
-: key(OLINK(from.key)), segs(from.segs)
+: key(OLINK(from.key)), filter(from.filter)
 {
     nodeKey = from.nodeKey;
     node.set(from.node);
@@ -1437,7 +1406,7 @@ void CKeyCursor::reset()
 {
     node.clear();
     matched = false;
-    eof = key.bloomFilterReject(*segs);
+    eof = key.bloomFilterReject(*filter) || !filter->canMatch();
     if (!eof)
         setLow(0);
 }
@@ -1709,7 +1678,7 @@ const byte *CKeyCursor::loadBlob(unsigned __int64 blobid, size32_t &blobsize)
 
 bool CKeyCursor::lookup(bool exact, KeyStatsCollector &stats)
 {
-    return _lookup(exact, segs->lastRealSeg(), stats);
+    return _lookup(exact, filter->lastRealSeg(), stats);
 }
 
 bool CKeyCursor::_lookup(bool exact, unsigned lastSeg, KeyStatsCollector &stats)
@@ -1735,16 +1704,7 @@ bool CKeyCursor::_lookup(bool exact, unsigned lastSeg, KeyStatsCollector &stats)
         if (!eof)
         {
             unsigned i = 0;
-            matched = true;
-            if (segs->segMonitors.length())
-            {
-                for (; i <= lastSeg; i++)
-                {
-                    matched = segs->segMonitors.item(i).matchesBuffer(keyBuffer);
-                    if (!matched)
-                        break;
-                }
-            }
+            matched = filter->matchesBuffer(keyBuffer, lastSeg, i);
             if (matched)
             {
                 ret = true;
@@ -1754,7 +1714,7 @@ bool CKeyCursor::_lookup(bool exact, unsigned lastSeg, KeyStatsCollector &stats)
             if (linuxYield)
                 sched_yield();
 #endif
-            eof = !segs->incrementKey(i, keyBuffer);
+            eof = !filter->incrementKey(i, keyBuffer);
             if (!exact)
             {
                 ret = true;
@@ -1805,7 +1765,7 @@ unsigned __int64 CKeyCursor::getCount(KeyStatsCollector &stats)
     reset();
     unsigned __int64 result = 0;
     unsigned lseeks = 0;
-    unsigned lastRealSeg = segs->lastRealSeg();
+    unsigned lastRealSeg = filter->lastRealSeg();
     for (;;)
     {
         if (_lookup(true, lastRealSeg, stats))
@@ -1830,7 +1790,7 @@ unsigned __int64 CKeyCursor::checkCount(unsigned __int64 max, KeyStatsCollector
     reset();
     unsigned __int64 result = 0;
     unsigned lseeks = 0;
-    unsigned lastFullSeg = segs->lastFullSeg();
+    unsigned lastFullSeg = filter->lastFullSeg();
     if (lastFullSeg == (unsigned) -1)
     {
         stats.noteSeeks(1, 0, 0);
@@ -1891,13 +1851,8 @@ void CKeyCursor::reportExcessiveSeeks(unsigned numSeeks, unsigned lastSeg, KeySt
     {
         recstr.appendf("%02x ", ((unsigned char *) keyBuffer)[i]);
     }
-    recstr.append ("\nusing segmonitors:\n");
-    for (i=0; i <= lastSeg; i++)
-    {
-        unsigned size = segs->segMonitors.item(i).getSize();
-        while (size--)
-            recstr.append( segs->segMonitors.item(i).isWild() ? '?' : '#');
-    }
+    recstr.append ("\nusing filter:\n");
+    filter->describe(recstr);
     if (stats.ctx)
         stats.ctx->CTXLOG("%d seeks to lookup record \n%s\n in key %s", numSeeks, recstr.str(), key.queryFileName());
     else
@@ -1931,17 +1886,9 @@ bool CKeyCursor::skipTo(const void *_seek, size32_t seekOffset, size32_t seeklen
     if (!seeklen) return false;
 
     unsigned j = setLowAfter(seekOffset + seeklen);
-    bool canmatch = true;
-    unsigned lastSeg = segs->lastRealSeg();
-    for (; j <= lastSeg; j++)
-    {
-        canmatch = segs->segMonitors.item(j).matchesBuffer(keyBuffer);
-        if (!canmatch)
-        {
-            eof = !incrementKey(j);
-            break;
-        }
-    }
+    bool canmatch = filter->matchesBuffer(keyBuffer, filter->lastRealSeg(), j);
+    if (!canmatch)
+        eof = !incrementKey(j);
     matched = false;
     return true;
 }
@@ -1954,14 +1901,229 @@ IKeyCursor * CKeyCursor::fixSortSegs(unsigned sortFieldOffset)
 CPartialKeyCursor::CPartialKeyCursor(const CKeyCursor &from, unsigned sortFieldOffset)
 : CKeyCursor(from)
 {
-    segs = new SegMonitorList(*segs, keyBuffer, sortFieldOffset);
+    filter = filter->fixSortSegs(keyBuffer, sortFieldOffset);
 }
 
 CPartialKeyCursor::~CPartialKeyCursor()
 {
-    ::Release(segs);
+    ::Release(filter);
+}
+
+//-------------------------------------------------------
+
+IndexRowFilter::IndexRowFilter(const RtlRecord &_recInfo) : recInfo(_recInfo)
+{
+    keySegCount = recInfo.getNumKeyedFields();
+    lastReal = 0;
+    lastFull = 0;
+    keyedSize = 0;
+}
+
+IndexRowFilter::IndexRowFilter(const IndexRowFilter &from, const char *fixedVals, unsigned sortFieldOffset)
+: recInfo(from.recInfo), keySegCount(from.keySegCount)
+{
+    lastReal = 0;
+    lastFull = 0;
+    keyedSize = 0;
+    ForEachItemIn(idx, from.filters)
+    {
+        auto &filter = from.filters.item(idx);
+        unsigned field = filter.queryFieldIndex();
+        unsigned offset = recInfo.getFixedOffset(field);
+        if (offset < sortFieldOffset)
+            append(FFkeyed, createFieldFilter(field, *recInfo.queryType(field), fixedVals+offset));
+        else
+            append(FFkeyed, LINK(&filter));  // MORE - FFopt vs FFkeyed is dodgy
+    }
+}
+
+
+void IndexRowFilter::append(IKeySegmentMonitor *segment)
+{
+    throwUnexpected();
+}
+
+const IIndexFilter *IndexRowFilter::item(unsigned idx) const
+{
+    return &queryFilter(idx);
+}
+
+void IndexRowFilter::append(FFoption option, const IFieldFilter * filter)
+{
+    assertex(filter->queryType().isFixedSize());
+    unsigned idx = filter->queryFieldIndex();
+    while (idx > numFilterFields())
+    {
+        append(FFkeyed, createWildFieldFilter(numFilterFields(), *recInfo.queryType(numFilterFields())));
+    }
+    assertex(idx == numFilterFields());
+    if (!filter->isWild())
+    {
+        lastReal = idx;
+        if (option != FFopt || lastFull == idx-1)
+            lastFull = idx;
+    }
+    keyedSize += filter->queryType().getMinSize();
+    addFilter(*filter);
+}
+
+void IndexRowFilter::setLow(unsigned field, void *keyBuffer) const
+{
+    unsigned lim = numFilterFields();
+    while (field < lim)
+    {
+        unsigned offset = recInfo.getFixedOffset(field);
+        const IFieldFilter &filter = queryFilter(field);
+        filter.setLow(keyBuffer, offset);
+        field++;
+    }
+}
+
+unsigned IndexRowFilter::setLowAfter(size32_t offset, void *keyBuffer) const
+{
+    unsigned lim = filters.length();
+    unsigned field = 0;
+    unsigned skipped = 0;
+    unsigned fieldOffset = recInfo.getFixedOffset(field);
+    while (field < lim)
+    {
+        unsigned nextOffset = recInfo.getFixedOffset(field+1);
+        if (fieldOffset >= offset)
+            filters.item(field).setLow(keyBuffer, fieldOffset);
+        else if (nextOffset <= offset)
+            skipped++;
+        else
+        {
+            byte *temp = (byte *) alloca(nextOffset - fieldOffset);
+            filters.item(field).setLow(temp, 0);
+            memcpy((byte *)keyBuffer+offset, temp, nextOffset - offset);
+        }
+        field++;
+        fieldOffset = nextOffset;
+    }
+    return skipped;
+}
+
+bool IndexRowFilter::incrementKey(unsigned segno, void *keyBuffer) const
+{
+    // Increment the key buffer to next acceptable value
+    for(;;)
+    {
+        if (queryFilter(segno).incrementKey(keyBuffer, recInfo.getFixedOffset(segno)))
+        {
+            setLow(segno+1, keyBuffer);
+            return true;
+        }
+        if (!segno)
+            return false;
+        segno--;
+    }
+}
+
+void IndexRowFilter::endRange(unsigned field, void *keyBuffer) const
+{
+    unsigned lim = numFilterFields();
+    if (field < lim)
+    {
+        queryFilter(field).endRange(keyBuffer, recInfo.getFixedOffset(field));
+        field++;
+    }
+    while (field < lim)
+    {
+        queryFilter(field).setHigh(keyBuffer, recInfo.getFixedOffset(field));
+        field++;
+    }
+}
+
+unsigned IndexRowFilter::lastRealSeg() const
+{
+    return lastReal;
+}
+
+unsigned IndexRowFilter::lastFullSeg() const
+{
+    return lastFull;
+}
+
+unsigned IndexRowFilter::numFilterFields() const
+{
+    return RowFilter::numFilterFields();
+}
+
+IIndexFilterList *IndexRowFilter::fixSortSegs(const char *fixedVals, unsigned sortFieldOffset) const
+{
+    return new IndexRowFilter(*this, fixedVals, sortFieldOffset);
+}
+
+void IndexRowFilter::reset()
+{
+    RowFilter::clear();
+    lastReal = 0;
+    lastFull = 0;
+    keyedSize = 0;
+}
+
+void IndexRowFilter::checkSize(size32_t _keyedSize, char const * keyname) const
+{
+    if (_keyedSize != keyedSize)
+    {
+        StringBuffer err;
+        err.appendf("Key size mismatch on key %s - key size is %u, expected %u", keyname, _keyedSize, keyedSize);
+    }
+}
+
+void IndexRowFilter::recalculateCache()
+{
+    // Nothing to do. This probably should be moved to be local to SegMonitorList
+}
+
+void IndexRowFilter::finish(size32_t _keyedSize)
+{
+    while (numFilterFields() < keySegCount)
+    {
+        unsigned idx = numFilterFields();
+        append(FFkeyed, createWildFieldFilter(idx, *recInfo.queryType(idx)));
+    }
+}
+
+void IndexRowFilter::describe(StringBuffer &out) const
+{
+    for (unsigned idx=0; idx < lastRealSeg(); idx++)
+    {
+        auto &filter = queryFilter(idx);
+        if (idx)
+            out.append(',');
+        out.appendf("%s=", recInfo.queryName(idx));
+        filter.describe(out);
+    }
 }
 
+bool IndexRowFilter::matchesBuffer(const void *buffer, unsigned lastSeg, unsigned &matchSeg) const
+{
+    if (numFilterFields())
+    {
+        RtlFixedRow rowInfo(recInfo, buffer, numFilterFields());
+        for (; matchSeg <= lastSeg; matchSeg++)
+        {
+            if (!queryFilter(matchSeg).matches(rowInfo))
+                return false;
+        }
+    }
+    return true;
+}
+
+bool IndexRowFilter::canMatch() const
+{
+    ForEachItemIn(idx, filters)
+    {
+        if (filters.item(idx).isEmpty())
+            return false;
+    }
+    return true;
+}
+
+//-------------------------------------------------------
+
 class CLazyKeyIndex : implements IKeyIndex, public CInterface
 {
     StringAttr keyfile;
@@ -2003,7 +2165,7 @@ public:
 
     virtual bool IsShared() const { return CInterface::IsShared(); }
 
-    virtual IKeyCursor *getCursor(const SegMonitorList *segs) override { return checkOpen().getCursor(segs); }
+    virtual IKeyCursor *getCursor(const IIndexFilterList *filter) override { return checkOpen().getCursor(filter); }
     virtual size32_t keySize() { return checkOpen().keySize(); }
     virtual size32_t keyedSize() { return checkOpen().keyedSize(); }
     virtual bool hasPayload() { return checkOpen().hasPayload(); }
@@ -2336,10 +2498,10 @@ class CKeyMerger : public CKeyLevelManager
     {
         // Make sure that sortFromSeg is properly set
         sortFromSeg = (unsigned) -1;
-        ForEachItemIn(idx, segs.segMonitors)
+        unsigned numFilters = filter->numFilterFields();
+        for (unsigned idx = 0; idx < numFilters; idx++)
         {
-            IKeySegmentMonitor &seg = segs.segMonitors.item(idx);
-            unsigned offset = seg.getOffset();
+            unsigned offset = filter->getFieldOffset(idx);
             if (offset == sortFieldOffset)
             {
                 sortFromSeg = idx;
@@ -2352,13 +2514,13 @@ class CKeyMerger : public CKeyLevelManager
     }
 
 public:
-    CKeyMerger(const RtlRecord &_recInfo, IKeyIndexSet *_keyset, unsigned _sortFieldOffset, IContextLogger *_ctx) : CKeyLevelManager(_recInfo, NULL, _ctx), sortFieldOffset(_sortFieldOffset)
+    CKeyMerger(const RtlRecord &_recInfo, IKeyIndexSet *_keyset, unsigned _sortFieldOffset, IContextLogger *_ctx, bool _newFilters) : CKeyLevelManager(_recInfo, NULL, _ctx, _newFilters), sortFieldOffset(_sortFieldOffset)
     {
         init();
         setKey(_keyset);
     }
 
-    CKeyMerger(const RtlRecord &_recInfo, IKeyIndex *_onekey, unsigned _sortFieldOffset, IContextLogger *_ctx) : CKeyLevelManager(_recInfo, NULL, _ctx), sortFieldOffset(_sortFieldOffset)
+    CKeyMerger(const RtlRecord &_recInfo, IKeyIndex *_onekey, unsigned _sortFieldOffset, IContextLogger *_ctx, bool _newFilters) : CKeyLevelManager(_recInfo, NULL, _ctx, _newFilters), sortFieldOffset(_sortFieldOffset)
     {
         init();
         setKey(_onekey);
@@ -2529,11 +2691,11 @@ public:
     void resetSort(const void *seek, size32_t seekOffset, size32_t seeklen)
     {
         activekeys = 0;
-        segs.recalculateCache();
+        filter->recalculateCache();
         unsigned i;
         for (i = 0; i < numkeys; i++)
         {
-            keyCursor = keyset->queryPart(i)->getCursor(&segs);
+            keyCursor = keyset->queryPart(i)->getCursor(filter);
             keyCursor->reset();
             for (;;)
             {
@@ -2611,7 +2773,7 @@ public:
         if (!started)
         {
             started = true;
-            segs.checkSize(keyedSize, "[merger]"); //PG: not sure what keyname to use here
+            filter->checkSize(keyedSize, "[merger]"); //PG: not sure what keyname to use here
         }
         if (!crappyHack)
         {
@@ -2738,7 +2900,7 @@ public:
             unsigned keyno;
             mb.read(keyno);
             keyNoArray.append(keyno);
-            keyCursor = keyset->queryPart(keyno)->getCursor(&segs);
+            keyCursor = keyset->queryPart(keyno)->getCursor(filter);
             keyCursor->deserializeCursorPos(mb, stats);
             cursorArray.append(*keyCursor);
             mergeHeapArray.append(i);
@@ -2752,20 +2914,20 @@ public:
         CKeyLevelManager::finishSegmentMonitors();
         if (sortFieldOffset)
         {
-            segs.checkSize(keyedSize, "[merger]"); // Ensures trailing KSM is setup
+            filter->checkSize(keyedSize, "[merger]"); // Ensures trailing KSM is setup
             calculateSortSeg();
         }
     }
 };
 
-extern jhtree_decl IKeyManager *createKeyMerger(const RtlRecord &_recInfo, IKeyIndexSet * _keys, unsigned _sortFieldOffset, IContextLogger *_ctx)
+extern jhtree_decl IKeyManager *createKeyMerger(const RtlRecord &_recInfo, IKeyIndexSet * _keys, unsigned _sortFieldOffset, IContextLogger *_ctx, bool _newFilters)
 {
-    return new CKeyMerger(_recInfo, _keys, _sortFieldOffset, _ctx);
+    return new CKeyMerger(_recInfo, _keys, _sortFieldOffset, _ctx, _newFilters);
 }
 
-extern jhtree_decl IKeyManager *createSingleKeyMerger(const RtlRecord &_recInfo, IKeyIndex * _onekey, unsigned _sortFieldOffset, IContextLogger *_ctx)
+extern jhtree_decl IKeyManager *createSingleKeyMerger(const RtlRecord &_recInfo, IKeyIndex * _onekey, unsigned _sortFieldOffset, IContextLogger *_ctx, bool _newFilters)
 {
-    return new CKeyMerger(_recInfo, _onekey, _sortFieldOffset, _ctx);
+    return new CKeyMerger(_recInfo, _onekey, _sortFieldOffset, _ctx, _newFilters);
 }
 
 class CKeyIndexSet : implements IKeyIndexSet, public CInterface
@@ -2793,9 +2955,9 @@ extern jhtree_decl IKeyIndexSet *createKeyIndexSet()
     return new CKeyIndexSet;
 }
 
-extern jhtree_decl IKeyManager *createLocalKeyManager(const RtlRecord &_recInfo, IKeyIndex *_key, IContextLogger *_ctx)
+extern jhtree_decl IKeyManager *createLocalKeyManager(const RtlRecord &_recInfo, IKeyIndex *_key, IContextLogger *_ctx, bool newFilters)
 {
-    return new CKeyLevelManager(_recInfo, _key, _ctx);
+    return new CKeyLevelManager(_recInfo, _key, _ctx, newFilters);
 }
 
 class CKeyArray : implements IKeyArray, public CInterface
@@ -2851,7 +3013,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
                                " { \"name\": \"f2\", \"type\": \"ty2\", \"flags\": 4 } ] "
                                "}";
             Owned<IOutputMetaData> meta = createTypeInfoOutputMetaData(json, false, nullptr);
-            Owned <IKeyManager> tlk1 = createKeyMerger(meta->queryRecordAccessor(true), keyset, 7, NULL);
+            Owned <IKeyManager> tlk1 = createKeyMerger(meta->queryRecordAccessor(true), keyset, 7, NULL, false);
             Owned<IStringSet> sset1 = createStringSet(7);
             sset1->addRange("0000003", "0000003");
             sset1->addRange("0000005", "0000006");
@@ -2888,7 +3050,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
             ASSERT(!tlk1->lookup(true)); 
             ASSERT(!tlk1->lookup(true)); 
 
-            Owned <IKeyManager> tlk2 = createKeyMerger(meta->queryRecordAccessor(true), NULL, 7, NULL);
+            Owned <IKeyManager> tlk2 = createKeyMerger(meta->queryRecordAccessor(true), NULL, 7, NULL, false);
             tlk2->setKey(keyset);
             tlk2->deserializeCursorPos(mb);
             tlk2->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
@@ -2904,7 +3066,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
             ASSERT(!tlk2->lookup(true)); 
             ASSERT(!tlk2->lookup(true)); 
 
-            Owned <IKeyManager> tlk3 = createKeyMerger(meta->queryRecordAccessor(true), NULL, 7, NULL);
+            Owned <IKeyManager> tlk3 = createKeyMerger(meta->queryRecordAccessor(true), NULL, 7, NULL, false);
             tlk3->setKey(keyset);
             tlk3->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
             tlk3->append(createKeySegmentMonitor(false, sset2.getLink(), 1, 7, 3));
@@ -2917,7 +3079,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
             ASSERT(!tlk3->lookupSkip("081", 7, 3)); 
             ASSERT(!tlk3->lookup(true)); 
 
-            Owned <IKeyManager> tlk4 = createKeyMerger(meta->queryRecordAccessor(true), NULL, 7, NULL);
+            Owned <IKeyManager> tlk4 = createKeyMerger(meta->queryRecordAccessor(true), NULL, 7, NULL, false);
             tlk4->setKey(keyset);
             tlk4->append(createKeySegmentMonitor(false, sset1.getLink(), 0, 0, 7));
             tlk4->append(createKeySegmentMonitor(false, sset3.getLink(), 1, 7, 3));
@@ -3047,14 +3209,14 @@ protected:
         buildTestKeys(variable);
         {
             Owned <IKeyIndex> index1 = createKeyIndex("keyfile1.$$$", 0, false, false);
-            Owned <IKeyManager> tlk1 = createLocalKeyManager(recInfo, index1, NULL);
+            Owned <IKeyManager> tlk1 = createLocalKeyManager(recInfo, index1, NULL, false);
             Owned<IStringSet> sset1 = createStringSet(10);
             sset1->addRange("0000000001", "0000000100");
             tlk1->append(createKeySegmentMonitor(false, sset1.getClear(), 0, 0, 10));
             tlk1->finishSegmentMonitors();
             tlk1->reset();
 
-            Owned <IKeyManager> tlk1a = createLocalKeyManager(recInfo, index1, NULL);
+            Owned <IKeyManager> tlk1a = createLocalKeyManager(recInfo, index1, NULL, false);
             Owned<IStringSet> sset1a = createStringSet(8);
             sset1a->addRange("00000000", "00000001");
             tlk1a->append(createKeySegmentMonitor(false, sset1a.getClear(), 0, 0, 8));
@@ -3078,7 +3240,7 @@ protected:
 
 
             Owned <IKeyIndex> index2 = createKeyIndex("keyfile2.$$$", 0, false, false);
-            Owned <IKeyManager> tlk2 = createLocalKeyManager(recInfo, index2, NULL);
+            Owned <IKeyManager> tlk2 = createLocalKeyManager(recInfo, index2, NULL, false);
             Owned<IStringSet> sset2 = createStringSet(10);
             sset2->addRange("0000000001", "0000000100");
             ASSERT(sset2->numValues() == 65536);
@@ -3093,7 +3255,7 @@ protected:
                 both->addIndex(index1.getLink());
                 both->addIndex(index2.getLink());
                 Owned<IStringSet> sset3 = createStringSet(10);
-                tlk3.setown(createKeyMerger(recInfo, NULL, 0, NULL));
+                tlk3.setown(createKeyMerger(recInfo, NULL, 0, NULL, false));
                 tlk3->setKey(both);
                 sset3->addRange("0000000001", "0000000100");
                 tlk3->append(createKeySegmentMonitor(false, sset3.getClear(), 0, 0, 10));
@@ -3101,7 +3263,7 @@ protected:
                 tlk3->reset();
             }
 
-            Owned <IKeyManager> tlk2a = createLocalKeyManager(recInfo, index2, NULL);
+            Owned <IKeyManager> tlk2a = createLocalKeyManager(recInfo, index2, NULL, false);
             Owned<IStringSet> sset2a = createStringSet(10);
             sset2a->addRange("0000000048", "0000000048");
             ASSERT(sset2a->numValues() == 1);
@@ -3109,7 +3271,7 @@ protected:
             tlk2a->finishSegmentMonitors();
             tlk2a->reset();
 
-            Owned <IKeyManager> tlk2b = createLocalKeyManager(recInfo, index2, NULL);
+            Owned <IKeyManager> tlk2b = createLocalKeyManager(recInfo, index2, NULL, false);
             Owned<IStringSet> sset2b = createStringSet(10);
             sset2b->addRange("0000000047", "0000000049");
             ASSERT(sset2b->numValues() == 3);
@@ -3117,7 +3279,7 @@ protected:
             tlk2b->finishSegmentMonitors();
             tlk2b->reset();
 
-            Owned <IKeyManager> tlk2c = createLocalKeyManager(recInfo, index2, NULL);
+            Owned <IKeyManager> tlk2c = createLocalKeyManager(recInfo, index2, NULL, false);
             Owned<IStringSet> sset2c = createStringSet(10);
             sset2c->addRange("0000000047", "0000000047");
             tlk2c->append(createKeySegmentMonitor(false, sset2c.getClear(), 0, 0, 10));

system/jhtree/jhtree.hpp

@@ -30,7 +30,7 @@
 #include "errorlist.h"
 
 class BloomFilter;
-class SegMonitorList;
+class IIndexFilterList;
 
 interface jhtree_decl IDelayedFile : public IInterface
 {
@@ -90,7 +90,7 @@ interface jhtree_decl IKeyIndexBase : public IInterface
 
 interface jhtree_decl IKeyIndex : public IKeyIndexBase
 {
-    virtual IKeyCursor *getCursor(const SegMonitorList *segs) = 0;
+    virtual IKeyCursor *getCursor(const IIndexFilterList *filter) = 0;
     virtual size32_t keySize() = 0;
     virtual bool isFullySorted() = 0;
     virtual bool isTopLevelKey() = 0;
@@ -184,48 +184,52 @@ typedef wchar_t UChar;
 typedef unsigned short UChar;
 #endif //_WIN32
 #include "rtlkey.hpp"
+#include "rtlnewkey.hpp"
 #include "jmisc.hpp"
 
 class RtlRecord;
 interface IDynamicTransform;
 
-class jhtree_decl SegMonitorList : implements IInterface, implements IIndexReadContext, public CInterface
+class jhtree_decl SegMonitorList : public CInterfaceOf<IIndexFilterList>
 {
-    unsigned _lastRealSeg() const;
     unsigned cachedLRS = 0;
     bool modified = true;
-    bool needWild;
     const RtlRecord &recInfo;
     unsigned keySegCount;
-
-public:
-    IMPLEMENT_IINTERFACE_O;
-    SegMonitorList(const RtlRecord &_recInfo, bool _needWild);
-    SegMonitorList(const SegMonitorList &_from, const char *fixedVals, unsigned sortFieldOffset);
     IArrayOf<IKeySegmentMonitor> segMonitors;
 
-    void reset();
-    void swapWith(SegMonitorList &other);
-    void setLow(unsigned segno, void *keyBuffer) const;
-    unsigned setLowAfter(size32_t offset, void *keyBuffer) const;
-    bool incrementKey(unsigned segno, void *keyBuffer) const;
-    void endRange(unsigned segno, void *keyBuffer) const;
-    inline unsigned lastRealSeg() const { assertex(!modified); return cachedLRS; }
-    unsigned lastFullSeg() const;
-    bool matched(void *keyBuffer, unsigned &lastMatch) const;
     size32_t getSize() const;
-    bool isExact(unsigned length, unsigned start) const;  // Are corresponding bytes an exact match ?
-    void checkSize(size32_t keyedSize, char const * keyname);
-    void recalculateCache();
-    void finish(size32_t keyedSize);
-    void deserialize(MemoryBuffer &mb);
-    void serialize(MemoryBuffer &mb) const;
+    unsigned _lastRealSeg() const;
+    SegMonitorList(const SegMonitorList &_from, const char *fixedVals, unsigned sortFieldOffset);
+public:
+    SegMonitorList(const RtlRecord &_recInfo);
 
     // interface IIndexReadContext
     virtual void append(IKeySegmentMonitor *segment) override;
-    virtual unsigned ordinality() const override;
-    virtual IKeySegmentMonitor *item(unsigned i) const override;
+    virtual IIndexFilter *item(unsigned i) const override;
     virtual void append(FFoption option, const IFieldFilter * filter) override;
+
+    // interface IIndexFilterList
+
+    virtual void setLow(unsigned segno, void *keyBuffer) const override;
+    virtual unsigned setLowAfter(size32_t offset, void *keyBuffer) const override;
+    virtual bool incrementKey(unsigned segno, void *keyBuffer) const override;
+    virtual void endRange(unsigned segno, void *keyBuffer) const override;
+    virtual unsigned lastRealSeg() const override { assertex(!modified); return cachedLRS; }
+    unsigned lastFullSeg() const override;
+    virtual unsigned numFilterFields() const override { return segMonitors.length(); }
+    virtual IIndexFilterList *fixSortSegs(const char *fixedVals, unsigned sortFieldOffset) const override
+    {
+        return new SegMonitorList(*this, fixedVals, sortFieldOffset);
+    }
+    virtual void reset() override;
+    virtual void checkSize(size32_t keyedSize, char const * keyname) const override;
+    virtual void recalculateCache() override;
+    virtual void finish(size32_t keyedSize) override;
+    virtual void describe(StringBuffer &out) const override;
+    virtual bool matchesBuffer(const void *buffer, unsigned lastSeg, unsigned &matchSeg) const override;
+    virtual unsigned getFieldOffset(unsigned idx) const override { return recInfo.getFixedOffset(idx); }
+    virtual bool canMatch() const override;
 };
 
 interface IKeyManager : public IInterface, extends IIndexReadContext
@@ -254,12 +258,12 @@ interface IKeyManager : public IInterface, extends IIndexReadContext
     virtual void resetCounts() = 0;
 
     virtual void setLayoutTranslator(const IDynamicTransform * trans) = 0;
-    virtual void setSegmentMonitors(SegMonitorList &segmentMonitors) = 0;
-    virtual void deserializeSegmentMonitors(MemoryBuffer &mb) = 0;
     virtual void finishSegmentMonitors() = 0;
+    virtual void describeFilter(StringBuffer &out) const = 0;
 
     virtual bool lookupSkip(const void *seek, size32_t seekGEOffset, size32_t seeklen) = 0;
     virtual unsigned getPartition() = 0;  // Use PARTITION() to retrieve partno, if possible, or zero to mean read all
+
 };
 
 inline offset_t extractFpos(IKeyManager * manager)
@@ -272,9 +276,9 @@ inline offset_t extractFpos(IKeyManager * manager)
 
 class RtlRecord;
 
-extern jhtree_decl IKeyManager *createLocalKeyManager(const RtlRecord &_recInfo, IKeyIndex * _key, IContextLogger *ctx);
-extern jhtree_decl IKeyManager *createKeyMerger(const RtlRecord &_recInfo, IKeyIndexSet * _key, unsigned sortFieldOffset, IContextLogger *ctx);
-extern jhtree_decl IKeyManager *createSingleKeyMerger(const RtlRecord &_recInfo, IKeyIndex * _onekey, unsigned sortFieldOffset, IContextLogger *ctx);
+extern jhtree_decl IKeyManager *createLocalKeyManager(const RtlRecord &_recInfo, IKeyIndex * _key, IContextLogger *ctx, bool _newFilters);
+extern jhtree_decl IKeyManager *createKeyMerger(const RtlRecord &_recInfo, IKeyIndexSet * _key, unsigned sortFieldOffset, IContextLogger *ctx, bool _newFilters);
+extern jhtree_decl IKeyManager *createSingleKeyMerger(const RtlRecord &_recInfo, IKeyIndex * _onekey, unsigned sortFieldOffset, IContextLogger *ctx, bool _newFilters);
 
 class KLBlobProviderAdapter : implements IBlobProvider
 {

system/jhtree/jhtree.ipp

@@ -106,7 +106,7 @@ public:
     virtual bool IsShared() const { return CInterface::IsShared(); }
 
 // IKeyIndex impl.
-    virtual IKeyCursor *getCursor(const SegMonitorList *segs) override;
+    virtual IKeyCursor *getCursor(const IIndexFilterList *filter) override;
 
     virtual size32_t keySize();
     virtual bool hasPayload();
@@ -130,7 +130,7 @@ public:
     virtual offset_t queryMetadataHead();
     virtual IPropertyTree * getMetadata();
 
-    bool bloomFilterReject(const SegMonitorList &segs) const;
+    bool bloomFilterReject(const IIndexFilterList &segs) const;
 
     virtual unsigned getNodeSize() { return keyHdr->getNodeSize(); }
     virtual bool hasSpecialFileposition() const;
@@ -170,21 +170,20 @@ public:
     virtual CJHTreeNode *loadNode(offset_t offset);
 };
 
-class jhtree_decl CKeyCursor : public IKeyCursor, public CInterface
+class jhtree_decl CKeyCursor : public CInterfaceOf<IKeyCursor>
 {
 protected:
     CKeyIndex &key;
-    const SegMonitorList *segs;
+    const IIndexFilterList *filter;
     char *keyBuffer = nullptr;
     Owned<CJHTreeNode> node;
     unsigned int nodeKey;
 
     bool eof=false;
-    bool matched=false;
+    bool matched=false; //MORE - this should probably be renamed. It's tracking state from one call of lookup to the next.
 
 public:
-    IMPLEMENT_IINTERFACE;
-    CKeyCursor(CKeyIndex &_key, const SegMonitorList *segs);
+    CKeyCursor(CKeyIndex &_key, const IIndexFilterList *filter);
     ~CKeyCursor();
 
     virtual bool next(char *dst, KeyStatsCollector &stats) override;
@@ -218,19 +217,19 @@ protected:
 
     inline void setLow(unsigned segNo)
     {
-        segs->setLow(segNo, keyBuffer);
+        filter->setLow(segNo, keyBuffer);
     }
     inline unsigned setLowAfter(size32_t offset)
     {
-        return segs->setLowAfter(offset, keyBuffer);
+        return filter->setLowAfter(offset, keyBuffer);
     }
     inline bool incrementKey(unsigned segno) const
     {
-        return segs->incrementKey(segno, keyBuffer);
+        return filter->incrementKey(segno, keyBuffer);
     }
     inline void endRange(unsigned segno)
     {
-        segs->endRange(segno, keyBuffer);
+        filter->endRange(segno, keyBuffer);
     }
 };
 
@@ -240,4 +239,43 @@ public:
     CPartialKeyCursor(const CKeyCursor &from, unsigned sortFieldOffset);
     ~CPartialKeyCursor();
 };
+
+// Specialization of a RowFilter allowing us to use them from JHTree. Allowed to assume that all keyed fields are fixed size (for now!)
+
+class IndexRowFilter : public RowFilter, public CInterfaceOf<IIndexFilterList>
+{
+public:
+    IndexRowFilter(const RtlRecord &_recInfo);
+    virtual void append(IKeySegmentMonitor *segment) override;
+    virtual const IIndexFilter *item(unsigned idx) const override;
+    virtual void append(FFoption option, const IFieldFilter * filter) override;
+
+    virtual void setLow(unsigned segno, void *keyBuffer) const override;
+    virtual unsigned setLowAfter(size32_t offset, void *keyBuffer) const override;
+    virtual bool incrementKey(unsigned segno, void *keyBuffer) const override;
+    virtual void endRange(unsigned segno, void *keyBuffer) const override;
+    virtual unsigned lastRealSeg() const override;
+    virtual unsigned lastFullSeg() const override;
+    virtual unsigned numFilterFields() const override;
+    virtual IIndexFilterList *fixSortSegs(const char *fixedVals, unsigned sortFieldOffset) const override;
+    virtual void reset() override;
+    virtual void checkSize(size32_t keyedSize, char const * keyname) const override;
+    virtual void recalculateCache() override;
+    virtual void finish(size32_t keyedSize) override;
+    virtual void describe(StringBuffer &out) const override;
+    virtual bool matchesBuffer(const void *buffer, unsigned lastSeg, unsigned &matchSeg) const override;
+    virtual unsigned getFieldOffset(unsigned idx) const override { return recInfo.getFixedOffset(idx); }
+    virtual bool canMatch() const override;
+
+protected:
+    IndexRowFilter(const IndexRowFilter &_from, const char *fixedVals, unsigned sortFieldOffset);
+
+    const RtlRecord &recInfo;
+    unsigned lastReal = 0;
+    unsigned lastFull = 0;
+    unsigned keyedSize = 0;
+    unsigned keySegCount = 0;
+
+};
+
 #endif

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -105,19 +105,6 @@ public:
         }
     }
 
-    virtual unsigned ordinality() const
-    {
-        return segMonitors.length();
-    }
-
-    virtual IKeySegmentMonitor *item(unsigned idx) const
-    {
-        if (segMonitors.isItem(idx))
-            return &segMonitors.item(idx);
-        else
-            return NULL;
-    }
-
     virtual void start()
     {
         CDiskReadSlaveActivityBase::start();

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -2379,7 +2379,7 @@ class IndexDistributeSlaveActivity : public HashDistributeSlaveBase
         CKeyLookup(IndexDistributeSlaveActivity &_owner, IHThorKeyedDistributeArg *_helper, IKeyIndex *_tlk)
             : owner(_owner), helper(_helper), tlk(_tlk)
         {
-            tlkManager.setown(createLocalKeyManager(helper->queryIndexRecordSize()->queryRecordAccessor(true), tlk, nullptr));
+            tlkManager.setown(createLocalKeyManager(helper->queryIndexRecordSize()->queryRecordAccessor(true), tlk, nullptr, helper->hasNewSegmentMonitors()));
             numslaves = owner.queryContainer().queryJob().querySlaves();
         }
         unsigned hash(const void *data)

thorlcr/activities/indexread/thindexread.cpp

@@ -160,7 +160,7 @@ protected:
                 if (!keyIndex)
                     throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", index->queryLogicalName());
 
-                Owned <IKeyManager> tlk = createLocalKeyManager(indexBaseHelper->queryDiskRecordSize()->queryRecordAccessor(true), keyIndex, nullptr);
+                Owned <IKeyManager> tlk = createLocalKeyManager(indexBaseHelper->queryDiskRecordSize()->queryRecordAccessor(true), keyIndex, nullptr, indexBaseHelper->hasNewSegmentMonitors());
                 indexBaseHelper->createSegmentMonitors(tlk);
                 tlk->finishSegmentMonitors();
                 tlk->reset();

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -244,7 +244,7 @@ public:
                 part.getCrc(crc);
 
                 Owned<IKeyIndex> keyIndex = createKeyIndex(filePath, crc, *lfile, false, false);
-                klManager.setown(createLocalKeyManager(helper->queryDiskRecordSize()->queryRecordAccessor(true), keyIndex, nullptr));
+                klManager.setown(createLocalKeyManager(helper->queryDiskRecordSize()->queryRecordAccessor(true), keyIndex, nullptr, helper->hasNewSegmentMonitors()));
                 if ((localKey && partDescs.ordinality()>1) || seekGEOffset)
                 {
                     // use key merger
@@ -524,7 +524,7 @@ public:
         }
         if (keyIndexSet)
         {
-            keyMergerManager.setown(createKeyMerger(helper->queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet, seekGEOffset, nullptr));
+            keyMergerManager.setown(createKeyMerger(helper->queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet, seekGEOffset, nullptr, helper->hasNewSegmentMonitors()));
             ITranslator const *translator = translators.item(0);
             if (translator)
                 keyMergerManager->setLayoutTranslator(&translator->queryTranslator());

thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp

@@ -1254,7 +1254,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
 
         CKeyLocalLookup(CKeyedJoinSlave &_owner, const RtlRecord &_keyRecInfo) : owner(_owner), keyRecInfo(_keyRecInfo), indexReadFieldsRow(_owner.indexInputAllocator)
         {
-            tlkManager.setown(owner.keyHasTlk ? createLocalKeyManager(keyRecInfo, nullptr, nullptr) : nullptr);
+            tlkManager.setown(owner.keyHasTlk ? createLocalKeyManager(keyRecInfo, nullptr, nullptr, owner.helper->hasNewSegmentMonitors()) : nullptr);
             reset();
             owner.getKeyIndexes(partKeyIndexes);
             RecordTranslationMode translationMode = getTranslationMode(owner);
@@ -1265,7 +1265,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
                 Owned<IKeyIndexSet> partKeySet = createKeyIndexSet();
                 ForEachItemIn(i, partKeyIndexes)
                     partKeySet->addIndex(LINK(&partKeyIndexes.item(i)));
-                partManager.setown(createKeyMerger(owner.helper->queryIndexRecordSize()->queryRecordAccessor(true), partKeySet, 0, nullptr));
+                partManager.setown(createKeyMerger(owner.helper->queryIndexRecordSize()->queryRecordAccessor(true), partKeySet, 0, nullptr, owner.helper->hasNewSegmentMonitors()));
                 Owned<const ITranslator> translator = getLayoutTranslation(owner.helper->getFileName(), owner.indexParts.item(0), translationMode, owner.helper->queryIndexRecordSize(), projectedFormat, expectedFormatCrc);
                 if (translator)
                     partManager->setLayoutTranslator(&translator->queryTranslator());
@@ -1273,7 +1273,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
             }
             else
             {
-                partManager.setown(createLocalKeyManager(owner.helper->queryIndexRecordSize()->queryRecordAccessor(true), nullptr, nullptr));
+                partManager.setown(createLocalKeyManager(owner.helper->queryIndexRecordSize()->queryRecordAccessor(true), nullptr, nullptr, owner.helper->hasNewSegmentMonitors()));
                 getLayoutTranslations(translators, owner.helper->getFileName(), owner.indexParts, translationMode, owner.helper->queryIndexRecordSize(), projectedFormat, expectedFormatCrc);
             }
         }

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -878,7 +878,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
                     Owned<IKeyIndex> keyIndex = activity.createPartKeyIndex(partNo, copy);
                     partKeySet->addIndex(keyIndex.getClear());
                 }
-                keyManager.setown(createKeyMerger(helper->queryIndexRecordSize()->queryRecordAccessor(true), partKeySet, 0, nullptr));
+                keyManager.setown(createKeyMerger(helper->queryIndexRecordSize()->queryRecordAccessor(true), partKeySet, 0, nullptr, helper->hasNewSegmentMonitors()));
                 setupTranslation(0, *keyManager);
             }
             processRows(processing, 0, keyManager);
@@ -1711,7 +1711,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
         {
             IKeyIndex *tlkKeyIndex = &tlkKeyIndexes.item(i);
             const RtlRecord &keyRecInfo = helper->queryIndexRecordSize()->queryRecordAccessor(true);
-            Owned<IKeyManager> tlkManager = createLocalKeyManager(keyRecInfo, nullptr, nullptr);
+            Owned<IKeyManager> tlkManager = createLocalKeyManager(keyRecInfo, nullptr, nullptr, helper->hasNewSegmentMonitors());
             tlkManager->setKey(tlkKeyIndex);
             keyManagers.append(*tlkManager.getClear());
         }
@@ -1734,7 +1734,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
     IKeyManager *createPartKeyManager(unsigned partNo, unsigned copy)
     {
         Owned<IKeyIndex> keyIndex = createPartKeyIndex(partNo, copy);
-        return createLocalKeyManager(helper->queryIndexRecordSize()->queryRecordAccessor(true), keyIndex, nullptr);
+        return createLocalKeyManager(helper->queryIndexRecordSize()->queryRecordAccessor(true), keyIndex, nullptr, helper->hasNewSegmentMonitors());
     }
     const void *preparePendingLookupRow(void *row, size32_t maxSz, const void *lhsRow, size32_t keySz)
     {

thorlcr/slave/slavmain.cpp

@@ -404,7 +404,7 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
 
         IKeyManager *createKeyManager()
         {
-            return createLocalKeyManager(queryHelper()->queryIndexRecordSize()->queryRecordAccessor(true), keyIndex, nullptr);
+            return createLocalKeyManager(queryHelper()->queryIndexRecordSize()->queryRecordAccessor(true), keyIndex, nullptr, queryHelper()->hasNewSegmentMonitors());
         }
         inline IHThorKeyedJoinArg *queryHelper() const { return activityCtx->queryHelper(); }
     };

tools/backupnode/CMakeLists.txt

@@ -35,6 +35,7 @@ include_directories (
          ./../../common/remote
          ./../../dali/base
          ./../../system/jhtree
+         ./../../rtl/include
          ./../../rtl/eclrtl
          ./../../system/security/shared
     )

tools/dumpkey/dumpkey.cpp

@@ -194,7 +194,7 @@ int main(int argc, const char **argv)
                 {
                     writer.setown(new SimpleOutputWriter);
                     const RtlRecord &inrec = diskmeta->queryRecordAccessor(true);
-                    manager.setown(createLocalKeyManager(inrec, index, nullptr));
+                    manager.setown(createLocalKeyManager(inrec, index, nullptr, false));
                     size32_t minRecSize = 0;
                     if (globals->hasProp("fields"))
                     {