HPCC-18435 Implement new versions of segment monitors

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/fileview2/fileview.hpp

@@ -156,7 +156,6 @@ extern FILEVIEW_API IResultSetFactory * getResultSetFactory(const char * usernam
 extern FILEVIEW_API IResultSetFactory * getSecResultSetFactory(ISecManager *secmgr, ISecUser *secuser, const char * username, const char * password);
 
 //Formatting applied remotely, so it can be accessed between different operating systems...
-extern FILEVIEW_API IResultSetFactory * getRemoteResultSetFactory(const char * remoteServer, const char * username, const char * password);
 extern FILEVIEW_API int findResultSetColumn(const INewResultSet * results, const char * columnName);
 
 extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns=NULL);

ecl/hql/hqlutil.hpp

@@ -748,7 +748,7 @@ extern HQL_API IHqlExpression * convertSetToExpression(bool isAll, size32_t len,
 
 struct FieldTypeInfoStruct;
 interface IRtlFieldTypeDeserializer;
-class RtlTypeInfo;
+struct RtlTypeInfo;
 
 /*
  * Check whether an xpath contains any non-scalar elements (and is this unsuitable for use when generating ECL)

rtl/eclrtl/CMakeLists.txt

@@ -36,6 +36,7 @@ set (    SRCS
          rtldynfield.cpp 
          rtlint.cpp 
          rtlkey.cpp 
+         rtlnewkey.cpp
          rtlqstr.cpp 
          rtlrank.cpp 
          rtlfield.cpp 
@@ -54,6 +55,7 @@ set (    SRCS
          rtlfield.hpp
          rtlkey2.hpp
          rtlkey.hpp
+         rtlnewkey.hpp
          rtlread_imp.hpp
          rtlrecord.hpp
          rtlsize.hpp

rtl/eclrtl/eclhelper_dyn.cpp

@@ -29,20 +29,7 @@
 #include "rtlkey.hpp"
 
 //---------------------------------------------------------------------------
-static void readString(StringBuffer &out, const char * &in)
-{
-    for (;;)
-    {
-        char c = *in++;
-        if (!c)
-            throw MakeStringException(0, "Invalid filter - missing closing '");
-        if (c=='\'')
-            break;
-        if (c=='\\')
-            UNIMPLEMENTED;
-        out.append(c);
-    }
-}
+
 class ECLRTL_API CDynamicDiskReadArg : public CThorDiskReadArg
 {
 public:
@@ -136,52 +123,7 @@ public:
             MemoryBuffer lobuffer;
             MemoryBuffer hibuffer;
             Owned<IStringSet> filterSet = createStringSet(fieldSize);
-            while (*filter)
-            {
-                char startRange = *filter++;
-                if (startRange != '(' && startRange != '[')
-                    throw MakeStringException(0, "Invalid filter string: expected [ or ( at start of range");
-                // Now we expect a constant - type depends on type of field. Assume string or int for now
-                StringBuffer upperString, lowerString;
-                if (*filter=='\'')
-                {
-                    filter++;
-                    readString(lowerString, filter);
-                }
-                else
-                    UNIMPLEMENTED; // lowerInt = readInt(curFilter);
-                if (*filter == ',')
-                {
-                    filter++;
-                    if (*filter=='\'')
-                    {
-                        filter++;
-                        readString(upperString, filter);
-                    }
-                    else
-                        UNIMPLEMENTED; //upperInt = readInt(curFilter);
-                }
-                else
-                    upperString.set(lowerString);
-                char endRange = *filter++;
-                if (endRange != ')' && endRange != ']')
-                    throw MakeStringException(0, "Invalid filter string: expected ] or ) at end of range");
-                if (*filter==',')
-                    filter++;
-                else if (*filter)
-                    throw MakeStringException(0, "Invalid filter string: expected , between ranges");
-                MemoryBufferBuilder lobuilder(lobuffer.clear(), inrec->getMinRecordSize());
-                fieldType->buildUtf8(lobuilder, 0, inrec->queryField(fieldNum), lowerString.length(), lowerString.str());
-
-                MemoryBufferBuilder hibuilder(hibuffer.clear(), inrec->getMinRecordSize());
-                fieldType->buildUtf8(hibuilder, 0, inrec->queryField(fieldNum), upperString.length(), upperString.str());
-
-                filterSet->addRange(lobuffer.toByteArray(), hibuffer.toByteArray());
-                if (startRange=='(')
-                    filterSet->killRange(lobuffer.toByteArray(), lobuffer.toByteArray());
-                if (endRange==')')
-                    filterSet->killRange(hibuffer.toByteArray(), hibuffer.toByteArray());
-            }
+            deserializeSet(*filterSet, inrec->getMinRecordSize(), fieldType, filter);
             filters.append(*filterSet.getClear());
             filterOffsets.append(fieldOffset);
             flags |= TDRkeyed;

rtl/eclrtl/eclrtl.hpp

@@ -427,7 +427,7 @@ inline unsigned __int64 rtlReadUInt8(const void * data) { return *(unsigned __in
 ECLRTL_API unsigned __int64 rtlReadUInt(const void * data, unsigned length);
 
 //MORE: Change if reverse endian, or if alignment issues.
-inline size32_t rtlReadSize32t(void * data) { return *(const size32_t *)data; }
+inline size32_t rtlReadSize32t(const void * data) { return *(const size32_t *)data; }
 
 inline void rtlWriteInt1(void * data, unsigned value) { *(unsigned char *)data = value; }
 inline void rtlWriteInt2(void * data, unsigned value) { *(unsigned short *)data = value; }

rtl/eclrtl/rtlfield.hpp

@@ -32,6 +32,9 @@ The file rtldynfield contains classes which manage instances of these classes wh
 size32_t ECLRTL_API getMinSize(const RtlFieldInfo * const * fields);
 
 // A base implementation of RtlTypeInfo
+// base classes should always implement buildString and buildUtf8 - default implementations of buildInt/buildReal
+// are implemented in terms of them.
+// The helper function buildUtf8ViaString can be used to provide a simple implementation of buildUtf8
 struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
 {
     constexpr inline RtlTypeInfoBase(unsigned _fieldType, unsigned _length) : RtlTypeInfo(_fieldType, _length) {}
@@ -42,8 +45,6 @@ struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const override;
-    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
-    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
     virtual size32_t buildReal(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, double val) const override;
 
@@ -59,6 +60,10 @@ struct ECLRTL_API RtlTypeInfoBase : public RtlTypeInfo
 
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
     virtual void readAhead(IRowDeserializerSource & in) const override;
+
+protected:
+    size32_t buildUtf8ViaString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const;
+    void getUtf8ViaString(size32_t & resultLen, char * & result, const void * ptr) const;
 };
 
 //-------------------------------------------------------------------------------------------------------------------
@@ -70,11 +75,16 @@ struct ECLRTL_API RtlBoolTypeInfo : public RtlTypeInfoBase
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
+protected:
+    bool getBool(const void * ptr) const;
 };
 
 struct ECLRTL_API RtlRealTypeInfo : public RtlTypeInfoBase
@@ -84,6 +94,8 @@ struct ECLRTL_API RtlRealTypeInfo : public RtlTypeInfoBase
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildReal(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, double val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
@@ -91,6 +103,7 @@ struct ECLRTL_API RtlRealTypeInfo : public RtlTypeInfoBase
     virtual __int64 getInt(const void * ptr) const override;
     virtual double getReal(const void * ptr) const override;
     virtual bool isNumeric() const override { return true; }
+    virtual int compare(const byte * left, const byte * right) const override;
 
 private:
     inline double value(const void * self) const;
@@ -104,6 +117,8 @@ struct ECLRTL_API RtlIntTypeInfo : public RtlTypeInfoBase
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
 
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
@@ -114,6 +129,7 @@ struct ECLRTL_API RtlIntTypeInfo : public RtlTypeInfoBase
     virtual bool canTruncate() const override;
     virtual bool canExtend(char &fillChar) const override;
     virtual bool isNumeric() const override { return true; }
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase
@@ -123,6 +139,8 @@ struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
 
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
@@ -133,6 +151,7 @@ struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase
     virtual bool canTruncate() const override;
     virtual bool canExtend(char &fillChar) const override;
     virtual bool isNumeric() const override { return true; }
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
@@ -144,6 +163,8 @@ struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
 
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
@@ -154,6 +175,7 @@ struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
     virtual __int64 getInt(const void * ptr) const override;
     virtual double getReal(const void * ptr) const override;
     virtual bool isNumeric() const override { return true; }
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlStringTypeInfo : public RtlTypeInfoBase
@@ -176,6 +198,7 @@ struct ECLRTL_API RtlStringTypeInfo : public RtlTypeInfoBase
     virtual __int64 getInt(const void * ptr) const override;
     virtual bool canTruncate() const override { return isFixedSize(); }
     virtual bool canExtend(char &fillChar) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlDataTypeInfo : public RtlTypeInfoBase
@@ -187,6 +210,7 @@ struct ECLRTL_API RtlDataTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
@@ -196,6 +220,7 @@ struct ECLRTL_API RtlDataTypeInfo : public RtlTypeInfoBase
     virtual __int64 getInt(const void * ptr) const override;
     virtual bool canTruncate() const override { return isFixedSize(); }
     virtual bool canExtend(char &fillChar) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlVarStringTypeInfo : public RtlTypeInfoBase
@@ -207,6 +232,7 @@ struct ECLRTL_API RtlVarStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
@@ -215,6 +241,7 @@ struct ECLRTL_API RtlVarStringTypeInfo : public RtlTypeInfoBase
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
     virtual bool canExtend(char &fillChar) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlQStringTypeInfo : public RtlTypeInfoBase
@@ -226,6 +253,7 @@ struct ECLRTL_API RtlQStringTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
@@ -234,6 +262,7 @@ struct ECLRTL_API RtlQStringTypeInfo : public RtlTypeInfoBase
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
     virtual bool canExtend(char &fillChar) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlDecimalTypeInfo : public RtlTypeInfoBase
@@ -246,12 +275,14 @@ struct ECLRTL_API RtlDecimalTypeInfo : public RtlTypeInfoBase
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const override;
     virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
     virtual double getReal(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 
     size32_t calcSize() const;
 };
@@ -262,11 +293,14 @@ struct ECLRTL_API RtlCharTypeInfo : public RtlTypeInfoBase
     virtual void doDelete() const final override { delete this; }
 
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlUnicodeTypeInfo : public RtlTypeInfoBase
@@ -279,6 +313,7 @@ public:
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
     virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
@@ -287,6 +322,7 @@ public:
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 
     virtual const char * queryLocale() const override { return locale; }
 
@@ -303,6 +339,7 @@ public:
     virtual size32_t getMinSize() const override;
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
     virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
 
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
@@ -312,6 +349,7 @@ public:
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 
     virtual const char * queryLocale() const override { return locale; }
 
@@ -328,6 +366,7 @@ public:
     virtual size32_t getMinSize() const override;
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
     virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
@@ -336,6 +375,7 @@ public:
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 
     virtual const char * queryLocale() const override { return locale; }
 
@@ -353,6 +393,8 @@ struct ECLRTL_API RtlRecordTypeInfo : public RtlTypeInfoBase
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
     virtual size32_t buildNull(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
@@ -360,6 +402,7 @@ struct ECLRTL_API RtlRecordTypeInfo : public RtlTypeInfoBase
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
     virtual const RtlFieldInfo * const * queryFields() const override { return fields; }
     virtual bool isScalar() const override { return false; }
 };
@@ -368,6 +411,8 @@ struct ECLRTL_API RtlCompoundTypeInfo : public RtlTypeInfoBase
 {
     constexpr inline RtlCompoundTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo * _child) : RtlTypeInfoBase(_fieldType, _length), child(_child) {}
 
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
@@ -390,6 +435,7 @@ struct ECLRTL_API RtlSetTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
     virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 struct ECLRTL_API RtlRowTypeInfo : public RtlCompoundTypeInfo
@@ -403,6 +449,7 @@ struct ECLRTL_API RtlRowTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
     virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 
@@ -418,6 +465,7 @@ struct ECLRTL_API RtlDatasetTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
     virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 
@@ -436,6 +484,7 @@ struct ECLRTL_API RtlDictionaryTypeInfo : public RtlCompoundTypeInfo
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
     virtual void readAhead(IRowDeserializerSource & in) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 
@@ -447,6 +496,8 @@ struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
     virtual bool getCondition(const byte * selfrow) const = 0;
     virtual size32_t getMinSize() const override;
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
@@ -455,6 +506,7 @@ struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
     virtual bool isScalar() const override { return false; }
+    virtual int compare(const byte * left, const byte * right) const override;
 
     virtual const RtlFieldInfo * const * queryFields() const override { return fields; }
 };
@@ -465,6 +517,9 @@ struct ECLRTL_API RtlBitfieldTypeInfo : public RtlTypeInfoBase
     constexpr inline RtlBitfieldTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
     virtual void doDelete() const final override { delete this; }
 
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t getMinSize() const override;
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
@@ -472,8 +527,10 @@ struct ECLRTL_API RtlBitfieldTypeInfo : public RtlTypeInfoBase
     virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
+    virtual int compare(const byte * left, const byte * right) const override;
 
 protected:
+    size32_t getSize() const;
     __int64 signedValue(const void * self) const;
     unsigned __int64 unsignedValue(const void * self) const;
 };
@@ -485,6 +542,8 @@ struct ECLRTL_API RtlUnimplementedTypeInfo : public RtlTypeInfoBase
 
     virtual size32_t getMinSize() const override;
     virtual size32_t size(const byte * self, const byte * selfrow) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
     virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
     virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const override;
@@ -493,6 +552,7 @@ struct ECLRTL_API RtlUnimplementedTypeInfo : public RtlTypeInfoBase
     virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
     virtual __int64 getInt(const void * ptr) const override;
     virtual bool isScalar() const override { return false; }
+    virtual int compare(const byte * left, const byte * right) const override;
 };
 
 /*

rtl/eclrtl/rtlkey.cpp

@@ -22,6 +22,7 @@
 #include "rtlkey2.hpp"
 #include "eclrtl_imp.hpp"
 #include "rtlrecord.hpp"
+#include "rtlnewkey.hpp"
 
 #define KSM_SET             0x01
 #define KSM_WILD            0x02
@@ -2317,6 +2318,42 @@ ECLRTL_API IStringSet *deserializeStringSet(MemoryBuffer &mb)
     return NULL; // up to caller to check
 };
 
+//---------------------------------------------------------------------------------------------------------------------
+
+class LegacySetCreator : implements ISetCreator
+{
+public:
+    LegacySetCreator(IStringSet & _set, size32_t _minRecordSize, const RtlTypeInfo * _fieldType)
+    : set(_set), minRecordSize(_minRecordSize), fieldType(_fieldType) {}
+
+    virtual void addRange(TransitionMask lowerMask, const StringBuffer & lowerString, TransitionMask upperMask, const StringBuffer & upperString) override
+    {
+        MemoryBufferBuilder lobuilder(lobuffer.clear(), minRecordSize);
+        fieldType->buildUtf8(lobuilder, 0, nullptr, lowerString.length(), lowerString.str());
+
+        MemoryBufferBuilder hibuilder(hibuffer.clear(), minRecordSize);
+        fieldType->buildUtf8(hibuilder, 0, nullptr, upperString.length(), upperString.str());
+
+        set.addRange(lobuffer.toByteArray(), hibuffer.toByteArray());
+        if (!(lowerMask & CMPeq))
+            set.killRange(lobuffer.toByteArray(), lobuffer.toByteArray());
+        if (!(upperMask & CMPeq))
+            set.killRange(hibuffer.toByteArray(), hibuffer.toByteArray());
+    }
+
+protected:
+    IStringSet & set;
+    const RtlTypeInfo *fieldType;
+    size32_t minRecordSize;
+    MemoryBuffer lobuffer;
+    MemoryBuffer hibuffer;
+};
+
+void deserializeSet(IStringSet & set, size32_t minRecordSize, const RtlTypeInfo * fieldType, const char * filter)
+{
+    LegacySetCreator creator(set, minRecordSize, fieldType);
+    deserializeSet(creator, filter);
+}
 
 #ifdef _USE_CPPUNIT
 #include <cppunit/extensions/HelperMacros.h>

rtl/eclrtl/rtlkey.hpp

@@ -179,7 +179,6 @@ ECLRTL_API IKeySegmentMonitor *createSingleBigSignedKeySegmentMonitor(bool optio
 ECLRTL_API IKeySegmentMonitor *createSingleLittleSignedKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
 ECLRTL_API IKeySegmentMonitor *createSingleLittleKeySegmentMonitor(bool optional, unsigned offset, unsigned size, const void * value);
 
-class RtlRecord;
 //takes over ownership of base
 ECLRTL_API IKeySegmentMonitor *createNewVarOffsetKeySegmentMonitor(IKeySegmentMonitor * base, unsigned offset, unsigned fieldIdx);
 
@@ -187,5 +186,6 @@ ECLRTL_API IKeySegmentMonitor *createNewVarOffsetKeySegmentMonitor(IKeySegmentMo
 ECLRTL_API IKeySegmentMonitor *createTranslatedKeySegmentMonitor(IKeySegmentMonitor * base, unsigned offset, IKeySegmentFormatTranslator * translator);
 
 ECLRTL_API IKeySegmentMonitor *deserializeKeySegmentMonitor(MemoryBuffer &mb);
+ECLRTL_API void deserializeSet(IStringSet & set, size32_t minRecordSize, const RtlTypeInfo * fieldType, const char * filter);
 
 #endif

rtl/eclrtl/rtlnewkey.hpp

@@ -0,0 +1,164 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef RTLNEWKEY_INCL
+#define RTLNEWKEY_INCL
+#include "eclrtl.hpp"
+
+enum TransitionMask : byte
+{
+    CMPlt = 0x01,
+    CMPeq = 0x02,
+    CMPgt = 0x04,
+    CMPmin = 0x08,      //minimum possible value
+    CMPmax = 0x10,      //maximum possible value
+    CMPle = CMPlt | CMPeq,
+    CMPge = CMPgt | CMPeq,
+    CMPminmask = CMPgt|CMPmin,
+    CMPmaxmask = CMPlt|CMPmax,
+};
+BITMASK_ENUM(TransitionMask);
+
+class ValueTransition;
+interface RtlTypeInfo;
+class RtlRow;
+
+/*
+ * The IValueSet interface represents a set of ranges of values.
+ *
+ * The transitions always come in pairs - an upper and lower bound.  Each bound can be inclusive or exclusive.
+ */
+interface IValueSet : public IInterface
+{
+//The following methods are used for creating a valueset
+    virtual ValueTransition * createTransition(TransitionMask mask, unsigned __int64 value) const = 0;
+    virtual ValueTransition * createStringTransition(TransitionMask mask, size32_t len, const char * value) const = 0;
+    virtual ValueTransition * createUtf8Transition(TransitionMask mask, size32_t len, const char * value) const = 0;
+    virtual void addRange(ValueTransition * loval, ValueTransition * hival) = 0;
+    virtual void addAll() = 0;
+    virtual void killRange(ValueTransition * loval, ValueTransition * hival) = 0;
+    virtual void reset() = 0;
+    virtual void invertSet() = 0;
+    virtual void unionSet(const IValueSet *) = 0;
+    virtual void excludeSet(const IValueSet *) = 0;
+    virtual void intersectSet(const IValueSet *) = 0;
+    virtual StringBuffer & serialize(StringBuffer & out) const= 0;
+
+//The following methods are for use once the value set has been created.
+    virtual unsigned numRanges() const = 0;
+
+    //find the last range where the lower bound <= the field, returns 0 if the field matches the lower bound, > 0 otherwise.
+    //matchRange is set to the range number, set to numRanges if there is no possible match.  Uses a binary chop
+    virtual int findCandidateRange(const byte * field, unsigned & matchRange) const = 0;
+
+    //find the last range where the lower bound <= the field, returns 0 if the field matches the lower bound, > 0 otherwise.
+    //starts searching from curRange (which is likely to match).  Uses a sequential search.
+    virtual int checkCandidateRange(const byte * field, unsigned & curRange) const = 0;
+
+    // Does this field match any range?
+    virtual bool matches(const byte * field) const = 0;
+
+    // Does this field match this particular range?
+    virtual bool matches(const byte * field, unsigned range) const = 0;
+
+    virtual const RtlTypeInfo & queryType() const = 0;
+};
+extern ECLRTL_API IValueSet * createValueSet(const RtlTypeInfo & type);
+
+interface ISetCreator
+{
+public:
+    virtual void addRange(TransitionMask lowerMask, const StringBuffer & lower, TransitionMask upperMask, const StringBuffer & upperString) = 0;
+};
+
+/*
+ * Read the textual representation of a set.
+ * The format of the set is an optional comma-separated sequence of ranges.
+ * Each range is specified as paren lower, upper paren, where the paren is either ( or [ depending
+ * on whether the specified bound is inclusive or exclusive.
+ * If only one bound is specified then it is used for both upper and lower bound (only meaningful with [] )
+ *
+ * ( A means values > A - exclusive
+ * [ means values >= A - inclusive
+ * A ) means values < A - exclusive
+ * A ] means values <= A - inclusive
+ * For example:
+ * [A] matches just A
+ * (,A),(A,) matches all but A
+ * (A] of [A) are both empty ranges
+ * [A,B) means A*
+ * Values use the ECL syntax for constants. String constants are always utf8. Binary use d'xx' format (hexpairs)
+ *
+ * @param creator   The interface that wraps the set that is being created
+ * @param filter    The textual representation of the set.
+ */
+extern ECLRTL_API void deserializeSet(ISetCreator & creator, const char * filter);
+
+/*
+ * Read the textual representation of a set.
+ *
+ * @param set       The target set to be updated.
+ * @param filter    The textual representation of the set.
+ */
+extern ECLRTL_API void deserializeSet(IValueSet & set, const char * filter);
+
+
+/*
+ * This interface represents a filter on a particular field of the row - similar to a segment monitor.
+ *
+ * Example implementations include single value, sets of ranges, regex or wildcard
+ */
+interface IFieldFilter : public IInterface
+{
+public:
+//Simple row matching
+    virtual bool matches(const RtlRow & row) const = 0;
+
+    virtual int compareRow(const RtlRow & left, const RtlRow & right) const = 0;
+
+    //MORE to come to support index lookups.
+    virtual unsigned numRanges() const = 0;
+    virtual int findCandidateRange(const RtlRow & row, unsigned & matchRange) const = 0;
+    virtual int checkCandidateRange(const RtlRow & row, unsigned & curRange) const = 0;
+    virtual bool withinUpperRange(const RtlRow & row, unsigned range) const = 0; // is the row within the current upper limit?
+    virtual bool matches(const RtlRow & row, unsigned range) const = 0;
+};
+
+//More types of IFieldFilter to come later
+extern ECLRTL_API IFieldFilter * createFieldFilter(unsigned fieldId, IValueSet * values);
+extern ECLRTL_API IFieldFilter * createWildFieldFilter(unsigned fieldId);
+
+/*
+ * The RowFilter class represents a multiple-field filter of a row.
+ */
+class RowFilter
+{
+public:
+    void addFilter(IFieldFilter & filter);
+    bool matches(const RtlRow & row) const;
+
+    int compareRows(const RtlRow & left, const RtlRow & right) const;
+    unsigned numFilterFields() const { return filters.ordinality(); }
+    const IFieldFilter & queryFilter(unsigned i) const { return filters.item(i); }
+
+protected:
+    IArrayOf<IFieldFilter> filters; // for an index must be in field order, and all values present - more thought required
+};
+
+
+
+#endif

rtl/eclrtl/rtlrecord.cpp

@@ -413,14 +413,14 @@ void RtlRow::getUtf8(size32_t & resultLen, char * & result, unsigned field) cons
 
 void RtlRow::setRow(const void * _row)
 {
-    row = _row;
+    row = (const byte *)_row;
     if (_row)
         info.calcRowOffsets(variableOffsets, _row);
 }
 
 void RtlRow::setRow(const void * _row, unsigned _numFields)
 {
-    row = _row;
+    row = (const byte *)_row;
     if (_row)
         info.calcRowOffsets(variableOffsets, _row, _numFields);
 }

rtl/eclrtl/rtlrecord.hpp

@@ -272,13 +272,18 @@ public:
     void setRow(const void * _row);
     void setRow(const void * _row, unsigned _numFields);
 
-    const void *queryRow() const
+    const byte *queryRow() const
     {
         return row;
     }
+    const byte * queryField(unsigned field) const
+    {
+        return queryRow() + getOffset(field);
+    }
+    explicit operator bool() { return row != nullptr; }
 protected:
     const RtlRecord & info;
-    const void * row;
+    const byte * row;
     size_t * variableOffsets;       // [0 + 1 entry for each variable size field ]
 };
 

rtl/include/eclhelper.hpp

@@ -381,7 +381,7 @@ interface RtlITypeInfo
     virtual size32_t getMinSize() const = 0;
     virtual size32_t deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in, size32_t offset) const = 0;
     virtual void readAhead(IRowDeserializerSource & in) const = 0;
-
+    virtual int compare(const byte * left, const byte * right) const = 0;
 protected:
     ~RtlITypeInfo() = default;  // we can't use a virtual destructor as we want constexpr constructors.
 };

system/jlib/jlib.hpp

@@ -334,4 +334,11 @@ struct DynamicScopeCtx
 typedef CIArrayOf<StringAttrItem> StringAttrArray;
 typedef CIArrayOf<StringBufferItem> StringBufferArray;
 
+//These could be template definitions, but that would potentially affect too many classes.
+#define BITMASK_ENUM(X) \
+inline constexpr X operator | (X l, X r) { return (X)((unsigned)l | (unsigned)r); } \
+inline constexpr X operator ~ (X l) { return (X)(~(unsigned)l); } \
+inline X & operator |= (X & l, X r) { l = l | r; return l; } \
+inline X & operator &= (X & l, X r) { l = (X)(l & r); return l; }
+
 #endif

system/jlib/jstats.h

@@ -250,14 +250,6 @@ protected:
     stat_type maxValue;
 };
 
-//These could be template definitions, but that would potentially affect too many classes.
-//MORE: Would it be useful to move this to a common definition point?
-#define BITMASK_ENUM(X) \
-inline constexpr X operator | (X l, X r) { return (X)((unsigned)l | (unsigned)r); } \
-inline constexpr X operator ~ (X l) { return (X)(~(unsigned)l); } \
-inline X & operator |= (X & l, X r) { l = l | r; return l; } \
-inline X & operator &= (X & l, X r) { l = (X)(l & r); return l; }
-
 enum ScopeCompare : unsigned
 {
     SCunknown   = 0x0000,   //