Merge pull request #12115 from ghalliday/issue21390

HPCC-21390 Remove default implementation of virtual functions from eclhelper.hpp

Reviewed-By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/roxierow.cpp

@@ -662,6 +662,7 @@ protected:
         virtual void toXML(const byte * self, IXmlWriter & out) {}
         virtual unsigned getVersion() const { return 0; }
         virtual unsigned getMetaFlags() { return 0; }
+        virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
         virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
 
         virtual void destruct(byte * self) {}
@@ -670,6 +671,7 @@ protected:
         virtual ISourceRowPrefetcher * createDiskPrefetcher() { return NULL; }
         virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
         virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+        virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
         virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
         virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
         virtual const RtlRecord &queryRecordAccessor(bool expand) const { UNIMPLEMENTED; }

common/thorhelper/thorcommon.ipp

@@ -129,6 +129,7 @@ public:
     virtual void toXML(const byte * self, IXmlWriter & out) { }
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return std::is_pod<T>() ? 0 : MDFneeddestruct; }
+    virtual const RtlTypeInfo * queryTypeInfo() const       { return nullptr; }
     virtual void destruct(byte * self)                      { reinterpret_cast<T *>(self)->~T(); }
     virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
@@ -136,6 +137,7 @@ public:
     virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
     virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
     virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
     virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); } // could provide a static implementation if needed
@@ -352,6 +354,7 @@ public:
 
     virtual void toXML(const byte * self, IXmlWriter & out)  { original->toXML(self+offset, out); }
     virtual unsigned getVersion() const { return original->getVersion(); }
+    virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
 
     virtual unsigned getMetaFlags() { return original->getMetaFlags(); }
     virtual void destruct(byte * self) { original->destruct(self+offset); }
@@ -381,6 +384,9 @@ public:
     {
         return new CPrefixedRowDeserializer(offset, original->createInternalDeserializer(ctx, activityId));
     }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to)
+    {
+    }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor)
     {
         original->walkIndirectMembers(self+offset, visitor);
@@ -490,6 +496,7 @@ public:
 
     virtual void toXML(const byte * self, IXmlWriter & out)  { original->toXML(self, out); }
     virtual unsigned getVersion() const { return original->getVersion(); }
+    virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
 
     virtual unsigned getMetaFlags() { return original->getMetaFlags(); }
     virtual void destruct(byte * self) { original->destruct(self); }
@@ -519,6 +526,9 @@ public:
     {
         return new CSuffixedRowDeserializer(offset, original->createInternalDeserializer(ctx, activityId));
     }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to)
+    {
+    }
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor)
     {
         original->walkIndirectMembers(self, visitor);

ecl/eclagent/eclagent.ipp

@@ -697,6 +697,7 @@ public:
     virtual void toXML(const byte * self, IXmlWriter & out) { }
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
+    virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
     virtual void destruct(byte * self) {}
     virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
@@ -704,6 +705,7 @@ public:
     virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
     virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
     virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
     virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); }  // Could be implemented if needed

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10836,7 +10836,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
         if (csvAttr)
             instance->addBaseClass("IHThorCsvWriteExtra", true);
         else if (xmlAttr)
-            instance->addBaseClass("IHThorXmlWriteExtra", true);
+            instance->addBaseClass("CHThorXmlWriteExtra", true); // CHThor... contains default implementations
     }
     instance->generateMetaFromInput = true;
 
@@ -11759,7 +11759,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityPipeThrough(BuildCtx & ctx, IH
     if (csvToPipe)
         instance->addBaseClass("IHThorCsvWriteExtra", true);
     else if (xmlToPipe)
-        instance->addBaseClass("IHThorXmlWriteExtra", true);
+        instance->addBaseClass("CHThorXmlWriteExtra", true); // CHThor... contains default implementations
     buildActivityFramework(instance);
 
     buildInstancePrefix(instance);

ecl/hthor/hthor.cpp

@@ -9772,6 +9772,7 @@ public:
     virtual void toXML(const byte * self, IXmlWriter & out) { }
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
+    virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
     virtual void destruct(byte * self)  {}
     virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
@@ -9779,6 +9780,7 @@ public:
     virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
     virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
     virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
     virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); } // could provide a static implementation if needed

roxie/ccd/ccdserver.cpp

@@ -15713,6 +15713,7 @@ public:
     virtual void toXML(const byte * self, IXmlWriter & out) { }
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
+    virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
     virtual void destruct(byte * self)  {}
     virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
@@ -15720,6 +15721,7 @@ public:
     virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
     virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
     virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
     virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); } // could provide a static implementation if needed
@@ -28028,6 +28030,7 @@ public:
     virtual void toXML(const byte * self, IXmlWriter & out) {}
     virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags()                         { return 0; }
+    virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
     virtual void destruct(byte * self)  {}
     virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
@@ -28035,6 +28038,7 @@ public:
     virtual IOutputMetaData * querySerializedDiskMeta() { return NULL; }
     virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
     virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
     virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
     virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); }

rtl/eclrtl/eclhelper_base.cpp

@@ -153,6 +153,9 @@ bool CThorCombineArg::canFilter() { return false; }
 
 bool CThorCombineGroupArg::canFilter() { return false; }
 
+// CThorRollupGroupArg
+bool CThorRollupGroupArg::canFilter() { return false; }
+
 //CThorActionArg
 
 void CThorActionArg::action() {}
@@ -352,6 +355,11 @@ IOutputMetaData * CThorKeyedJoinArg::queryDiskRecordSize() { return NULL; }
 IOutputMetaData * CThorKeyedJoinArg::queryProjectedDiskRecordSize() { return NULL; }
 unsigned __int64 CThorKeyedJoinArg::extractPosition(const void * _right) { return 0; }
 
+unsigned CThorKeyedJoinArg::getFetchFlags() { return 0; }
+unsigned CThorKeyedJoinArg::getDiskFormatCrc() { return 0; }
+unsigned CThorKeyedJoinArg::getProjectedFormatCrc() { return getDiskFormatCrc(); }  // Should really be crc of queryProjectedDiskRecordSize layout
+void CThorKeyedJoinArg::getFileEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
+
 bool CThorKeyedJoinArg::leftCanMatch(const void * inputRow) { return true; }
 bool CThorKeyedJoinArg::indexReadMatch(const void * indexRow, const void * inputRow, IBlobProvider * blobs) { return true; }
 
@@ -360,6 +368,8 @@ void CThorKeyedJoinArg::onLimitExceeded() { }
 unsigned __int64 CThorKeyedJoinArg::getSkipLimit() { return 0; }
 unsigned CThorKeyedJoinArg::getMatchAbortLimit() { return 0; }
 void CThorKeyedJoinArg::onMatchAbortLimitExceeded() { }
+unsigned CThorKeyedJoinArg::getProjectedIndexFormatCrc() { return getIndexFormatCrc(); }  // Should really be crc of queryIndexReadInputRecordSize layout (?)
+
 
 unsigned CThorKeyedJoinArg::getJoinLimit() { return 0; }
 unsigned CThorKeyedJoinArg::getKeepLimit() { return 0; }
@@ -526,9 +536,39 @@ const char * CThorCsvWriteArg::getCluster(unsigned idx) { return NULL; }
 
 bool CThorXmlParseArg::requiresContents() { return false; }
 
+
+unsigned CThorFetchArg::getFetchFlags() { return 0; }
+unsigned CThorFetchArg::getDiskFormatCrc() { return 0; }
+unsigned CThorFetchArg::getProjectedFormatCrc() { return getDiskFormatCrc(); }  // Should really be crc of queryProjectedDiskRecordSize layout
+void CThorFetchArg::getFileEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
+unsigned __int64 CThorFetchArg::getRowLimit() { return (unsigned __int64) -1; }
+void CThorFetchArg::onLimitExceeded()         { }
+size32_t CThorFetchArg::extractJoinFields(ARowBuilder & rowBuilder, const void * _right) { return 0; }
+bool CThorFetchArg::extractAllJoinFields()    { return false; }
+IOutputMetaData * CThorFetchArg::queryExtractedSize() { return NULL; }
+
+unsigned CThorCsvFetchArg::getFetchFlags() { return 0; }
+unsigned CThorCsvFetchArg::getDiskFormatCrc() { return 0; }
+unsigned CThorCsvFetchArg::getProjectedFormatCrc() { return getDiskFormatCrc(); }  // Should really be crc of queryProjectedDiskRecordSize layout
+void CThorCsvFetchArg::getFileEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
+unsigned __int64 CThorCsvFetchArg::getRowLimit() { return (unsigned __int64) -1; }
+void CThorCsvFetchArg::onLimitExceeded()         { }
+size32_t CThorCsvFetchArg::extractJoinFields(ARowBuilder & rowBuilder, const void * _right) { return 0; }
+bool CThorCsvFetchArg::extractAllJoinFields()    { return false; }
+IOutputMetaData * CThorCsvFetchArg::queryExtractedSize() { return NULL; }
+
 //CThorXmlFetchArg
 
+unsigned CThorXmlFetchArg::getFetchFlags() { return 0; }
+unsigned CThorXmlFetchArg::getDiskFormatCrc() { return 0; }
+unsigned CThorXmlFetchArg::getProjectedFormatCrc() { return getDiskFormatCrc(); }  // Should really be crc of queryProjectedDiskRecordSize layout
+void CThorXmlFetchArg::getFileEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
 bool CThorXmlFetchArg::requiresContents() { return false; }
+unsigned __int64 CThorXmlFetchArg::getRowLimit() { return (unsigned __int64) -1; }
+void CThorXmlFetchArg::onLimitExceeded()         { }
+size32_t CThorXmlFetchArg::extractJoinFields(ARowBuilder & rowBuilder, const void * _right) { return 0; }
+bool CThorXmlFetchArg::extractAllJoinFields()    { return false; }
+IOutputMetaData * CThorXmlFetchArg::queryExtractedSize() { return NULL; }
 
 //CThorXmlWriteArg
 
@@ -545,6 +585,12 @@ void CThorXmlWriteArg::getUpdateCRCs(unsigned & eclCRC, unsigned __int64 & total
 void CThorXmlWriteArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
 const char * CThorXmlWriteArg::getCluster(unsigned idx) { return NULL; }
 
+//CHThorXmlWriteExtra
+const char * CHThorXmlWriteExtra::getXmlIteratorPath() { return NULL; }             // supplies the prefix and suffix for a row
+const char * CHThorXmlWriteExtra::getHeader() { return NULL; }
+const char * CHThorXmlWriteExtra::getFooter() { return NULL; }
+unsigned CHThorXmlWriteExtra::getXmlFlags() { return 0; }
+
 //CThorSoapActionArg
 
 void CThorSoapActionArg::toXML(const byte * self, IXmlWriter & out) { return; }
@@ -605,6 +651,15 @@ size32_t CThorParseArg::executeProduction(ARowBuilder & rowBuilder, unsigned id,
 unsigned CThorIndexReadArg::getFlags() { return 0; }
 bool CThorIndexReadArg::getIndexLayout(size32_t & _retLen, void * & _retData) { return false; }
 void CThorIndexReadArg::setCallback(IThorIndexCallback * _tc) { fpp = _tc; }
+bool CThorIndexReadArg::canMatchAny()                              { return true; }
+void CThorIndexReadArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorIndexReadArg::canMatch(const void * row)                 { return true; }
+bool CThorIndexReadArg::hasMatchFilter()                           { return false; }
+IHThorSteppedSourceExtra * CThorIndexReadArg::querySteppingExtra()  { return nullptr; }
+
+unsigned __int64 CThorIndexReadArg::getChooseNLimit()              { return I64C(0x7fffffffffffffff); }
+unsigned __int64 CThorIndexReadArg::getRowLimit()                  { return (unsigned __int64) -1; }
+void CThorIndexReadArg::onLimitExceeded()                          { }
 
 bool CThorIndexReadArg::needTransform() { return false; }
 bool CThorIndexReadArg::transformMayFilter() { return false; }
@@ -617,7 +672,6 @@ size32_t CThorIndexReadArg::unfilteredTransform(ARowBuilder & rowBuilder, const
 
 size32_t CThorIndexReadArg::transformOnLimitExceeded(ARowBuilder & rowBuilder) { return 0; }
 size32_t CThorIndexReadArg::transformOnKeyedLimitExceeded(ARowBuilder & rowBuilder) { return 0; }
-IHThorSteppedSourceExtra *CThorIndexReadArg::querySteppingExtra() { return NULL; }
 
 //CThorSteppedIndexReadArg
 
@@ -632,6 +686,17 @@ unsigned CThorSteppedIndexReadArg::getPrefetchSize() { return 0; }
 unsigned CThorIndexNormalizeArg::getFlags() { return 0; }
 bool CThorIndexNormalizeArg::getIndexLayout(size32_t & _retLen, void * & _retData) { return false; }
 void CThorIndexNormalizeArg::setCallback(IThorIndexCallback * _tc) { fpp = _tc; }
+bool CThorIndexNormalizeArg::canMatchAny()                              { return true; }
+void CThorIndexNormalizeArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorIndexNormalizeArg::canMatch(const void * row)                 { return true; }
+bool CThorIndexNormalizeArg::hasMatchFilter()                           { return false; }
+IHThorSteppedSourceExtra * CThorIndexNormalizeArg::querySteppingExtra()  { return nullptr; }
+
+unsigned __int64 CThorIndexNormalizeArg::getChooseNLimit()              { return I64C(0x7fffffffffffffff); }
+unsigned __int64 CThorIndexNormalizeArg::getRowLimit()                  { return (unsigned __int64) -1; }
+void CThorIndexNormalizeArg::onLimitExceeded()                          { }
+unsigned __int64 CThorIndexNormalizeArg::getKeyedLimit()                { return (unsigned __int64) -1; }
+void CThorIndexNormalizeArg::onKeyedLimitExceeded()                     { }
 
 size32_t CThorIndexNormalizeArg::transformOnLimitExceeded(ARowBuilder & rowBuilder) { return 0; }
 size32_t CThorIndexNormalizeArg::transformOnKeyedLimitExceeded(ARowBuilder & rowBuilder) { return 0; }
@@ -641,6 +706,11 @@ size32_t CThorIndexNormalizeArg::transformOnKeyedLimitExceeded(ARowBuilder & row
 unsigned CThorIndexAggregateArg::getFlags() { return 0; }
 bool CThorIndexAggregateArg::getIndexLayout(size32_t & _retLen, void * & _retData) { return false; }
 void CThorIndexAggregateArg::setCallback(IThorIndexCallback * _tc) { fpp = _tc; }
+bool CThorIndexAggregateArg::canMatchAny()                              { return true; }
+void CThorIndexAggregateArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorIndexAggregateArg::canMatch(const void * row)                 { return true; }
+bool CThorIndexAggregateArg::hasMatchFilter()                           { return false; }
+IHThorSteppedSourceExtra * CThorIndexAggregateArg::querySteppingExtra()  { return nullptr; }
 size32_t CThorIndexAggregateArg::mergeAggregate(ARowBuilder & rowBuilder, const void * src) { rtlFailUnexpected(); return 0; }
 void CThorIndexAggregateArg::processRows(ARowBuilder & rowBuilder, size32_t srcLen, const void * src) { rtlFailUnexpected(); }
 
@@ -649,6 +719,11 @@ void CThorIndexAggregateArg::processRows(ARowBuilder & rowBuilder, size32_t srcL
 unsigned CThorIndexCountArg::getFlags() { return 0; }
 bool CThorIndexCountArg::getIndexLayout(size32_t & _retLen, void * & _retData) { return false; }
 void CThorIndexCountArg::setCallback(IThorIndexCallback * _tc) { fpp = _tc; }
+bool CThorIndexCountArg::canMatchAny()                              { return true; }
+void CThorIndexCountArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorIndexCountArg::canMatch(const void * row)                 { return true; }
+bool CThorIndexCountArg::hasMatchFilter()                           { return false; }
+IHThorSteppedSourceExtra * CThorIndexCountArg::querySteppingExtra()  { return nullptr; }
 unsigned __int64 CThorIndexCountArg::getRowLimit() { return (unsigned __int64) -1; }
 void CThorIndexCountArg::onLimitExceeded() { }
 unsigned __int64 CThorIndexCountArg::getKeyedLimit() { return (unsigned __int64) -1; }
@@ -658,12 +733,20 @@ size32_t CThorIndexCountArg::numValid(size32_t srcLen, const void * _src)
     rtlFailUnexpected();
     return 0;
 }
+bool CThorIndexCountArg::hasFilter()                                                { return false; }       // also true if denormalized(!)
+size32_t CThorIndexCountArg::numValid(const void * src)                             { return 1; }           //NB: Can be > 1 if source is normlized
+unsigned __int64 CThorIndexCountArg::getChooseNLimit()                              { return (unsigned __int64) -1; }
 
 //CThorIndexGroupAggregateArg
 
 unsigned CThorIndexGroupAggregateArg::getFlags() { return 0; }
 bool CThorIndexGroupAggregateArg::getIndexLayout(size32_t & _retLen, void * & _retData) { return false; }
 void CThorIndexGroupAggregateArg::setCallback(IThorIndexCallback * _tc) { fpp = _tc; }
+bool CThorIndexGroupAggregateArg::canMatchAny()                              { return true; }
+void CThorIndexGroupAggregateArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorIndexGroupAggregateArg::canMatch(const void * row)                 { return true; }
+bool CThorIndexGroupAggregateArg::hasMatchFilter()                           { return false; }
+IHThorSteppedSourceExtra * CThorIndexGroupAggregateArg::querySteppingExtra()  { return nullptr; }
 bool CThorIndexGroupAggregateArg::createGroupSegmentMonitors(IIndexReadContext *ctx) { return false; }
 unsigned CThorIndexGroupAggregateArg::getGroupingMaxField() { return 0; }
 size32_t CThorIndexGroupAggregateArg::initialiseCountGrouping(ARowBuilder & rowBuilder, const void * src) { rtlFailUnexpected(); return 0; }
@@ -676,6 +759,15 @@ void CThorIndexGroupAggregateArg::processRows(size32_t srcLen, const void * src,
 
 unsigned CThorDiskReadArg::getFlags() { return 0; }
 void CThorDiskReadArg::setCallback(IThorDiskCallback * _tc) { fpp = _tc; }
+bool CThorDiskReadArg::canMatchAny()                              { return true; }
+void CThorDiskReadArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorDiskReadArg::canMatch(const void * row)                 { return true; }
+bool CThorDiskReadArg::hasMatchFilter()                           { return false; }
+void CThorDiskReadArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
+
+unsigned __int64 CThorDiskReadArg::getChooseNLimit()              { return I64C(0x7fffffffffffffff); }
+unsigned __int64 CThorDiskReadArg::getRowLimit()                  { return (unsigned __int64) -1; }
+void CThorDiskReadArg::onLimitExceeded()                          { }
 
 bool CThorDiskReadArg::needTransform() { return false; }
 bool CThorDiskReadArg::transformMayFilter() { return false; }
@@ -697,6 +789,16 @@ size32_t CThorDiskReadArg::transformOnKeyedLimitExceeded(ARowBuilder & rowBuilde
 
 unsigned CThorDiskNormalizeArg::getFlags() { return 0; }
 void CThorDiskNormalizeArg::setCallback(IThorDiskCallback * _tc) { fpp = _tc; }
+bool CThorDiskNormalizeArg::canMatchAny()                              { return true; }
+void CThorDiskNormalizeArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorDiskNormalizeArg::canMatch(const void * row)                 { return true; }
+bool CThorDiskNormalizeArg::hasMatchFilter()                           { return false; }
+void CThorDiskNormalizeArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
+unsigned __int64 CThorDiskNormalizeArg::getChooseNLimit()              { return I64C(0x7fffffffffffffff); }
+unsigned __int64 CThorDiskNormalizeArg::getRowLimit()                  { return (unsigned __int64) -1; }
+void CThorDiskNormalizeArg::onLimitExceeded()                          { }
+unsigned __int64 CThorDiskNormalizeArg::getKeyedLimit()                { return (unsigned __int64) -1; }
+void CThorDiskNormalizeArg::onKeyedLimitExceeded()                     { }
 size32_t CThorDiskNormalizeArg::transformOnLimitExceeded(ARowBuilder & rowBuilder) { return 0; }
 size32_t CThorDiskNormalizeArg::transformOnKeyedLimitExceeded(ARowBuilder & rowBuilder) { return 0; }
 
@@ -704,22 +806,40 @@ size32_t CThorDiskNormalizeArg::transformOnKeyedLimitExceeded(ARowBuilder & rowB
 
 unsigned CThorDiskAggregateArg::getFlags() { return 0; }
 void CThorDiskAggregateArg::setCallback(IThorDiskCallback * _tc) { fpp = _tc; }
+bool CThorDiskAggregateArg::canMatchAny()                              { return true; }
+void CThorDiskAggregateArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorDiskAggregateArg::canMatch(const void * row)                 { return true; }
+bool CThorDiskAggregateArg::hasMatchFilter()                           { return false; }
+void CThorDiskAggregateArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
 size32_t CThorDiskAggregateArg::mergeAggregate(ARowBuilder & rowBuilder, const void * src) { rtlFailUnexpected(); return 0; }
 
 //CThorDiskCountArg
 
 unsigned CThorDiskCountArg::getFlags() { return 0; }
 void CThorDiskCountArg::setCallback(IThorDiskCallback * _tc) { fpp = _tc; }
+bool CThorDiskCountArg::canMatchAny()                              { return true; }
+void CThorDiskCountArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorDiskCountArg::canMatch(const void * row)                 { return true; }
+bool CThorDiskCountArg::hasMatchFilter()                           { return false; }
+void CThorDiskCountArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
 
 unsigned __int64 CThorDiskCountArg::getRowLimit() { return (unsigned __int64) -1; }
 void CThorDiskCountArg::onLimitExceeded() { }
 unsigned __int64 CThorDiskCountArg::getKeyedLimit() { return (unsigned __int64) -1; }
 void CThorDiskCountArg::onKeyedLimitExceeded() { }
+bool CThorDiskCountArg::hasFilter()                                                { return false; }       // also true if denormalized(!)
+size32_t CThorDiskCountArg::numValid(const void * src)                             { return 1; }           //NB: Can be > 1 if source is normlized
+unsigned __int64 CThorDiskCountArg::getChooseNLimit()                              { return (unsigned __int64) -1; }
 
 //CThorDiskGroupAggregateArg
 
 unsigned CThorDiskGroupAggregateArg::getFlags() { return 0; }
 void CThorDiskGroupAggregateArg::setCallback(IThorDiskCallback * _tc) { fpp = _tc; }
+bool CThorDiskGroupAggregateArg::canMatchAny()                              { return true; }
+void CThorDiskGroupAggregateArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorDiskGroupAggregateArg::canMatch(const void * row)                 { return true; }
+bool CThorDiskGroupAggregateArg::hasMatchFilter()                           { return false; }
+void CThorDiskGroupAggregateArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
 bool CThorDiskGroupAggregateArg::createGroupSegmentMonitors(IIndexReadContext *ctx) { return false; }
 unsigned CThorDiskGroupAggregateArg::getGroupingMaxField() { return 0; }
 size32_t CThorDiskGroupAggregateArg::initialiseCountGrouping(ARowBuilder & rowBuilder, const void * src) { rtlFailUnexpected(); return 0; }
@@ -735,6 +855,11 @@ void CThorCsvReadArg::onLimitExceeded() { }
 unsigned CThorCsvReadArg::getDiskFormatCrc() { return 0; }   // no meaning
 unsigned CThorCsvReadArg::getProjectedFormatCrc() { return 0; }   // no meaning
 void CThorCsvReadArg::setCallback(IThorDiskCallback * _tc) { fpp = _tc; }
+bool CThorCsvReadArg::canMatchAny()                              { return true; }
+void CThorCsvReadArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorCsvReadArg::canMatch(const void * row)                 { return true; }
+bool CThorCsvReadArg::hasMatchFilter()                           { return false; }
+void CThorCsvReadArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
 
 //CThorXmlReadArg
 
@@ -745,11 +870,23 @@ void CThorXmlReadArg::onLimitExceeded() { }
 unsigned CThorXmlReadArg::getDiskFormatCrc() { return 0; }   // no meaning
 unsigned CThorXmlReadArg::getProjectedFormatCrc() { return 0; }   // no meaning
 void CThorXmlReadArg::setCallback(IThorDiskCallback * _tc) { fpp = _tc; }
+bool CThorXmlReadArg::canMatchAny()                              { return true; }
+void CThorXmlReadArg::createSegmentMonitors(IIndexReadContext *ctx) {}
+bool CThorXmlReadArg::canMatch(const void * row)                 { return true; }
+bool CThorXmlReadArg::hasMatchFilter()                           { return false; }
+void CThorXmlReadArg::getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
 
 //CThorChildGroupAggregateArg
 
 size32_t CThorChildGroupAggregateArg::mergeAggregate(ARowBuilder & rowBuilder, const void * src) { rtlFailUnexpected(); return 0; }
 
+//CThorChildThroughNormalizeArg
+unsigned __int64 CThorChildThroughNormalizeArg::getChooseNLimit()              { return I64C(0x7fffffffffffffff); }
+unsigned __int64 CThorChildThroughNormalizeArg::getRowLimit()                  { return (unsigned __int64) -1; }
+void CThorChildThroughNormalizeArg::onLimitExceeded()                          { }
+unsigned __int64 CThorChildThroughNormalizeArg::getKeyedLimit()                { return (unsigned __int64) -1; }
+void CThorChildThroughNormalizeArg::onKeyedLimitExceeded()                     { }
+
 //CThorLoopArg
 
 unsigned CThorLoopArg::getFlags() { return 0; }

rtl/eclrtl/rtlrecord.hpp

@@ -379,6 +379,7 @@ public:
     }
     virtual unsigned getVersion() const override                     { return OUTPUTMETADATA_VERSION; }
     virtual unsigned getMetaFlags() override                         { return MDFhasserialize|MDFhasxml; }
+    virtual const RtlTypeInfo * queryTypeInfo() const override { return nullptr; }
 
     virtual void destruct(byte * self) override                      {}
     virtual IOutputMetaData * querySerializedDiskMeta() override    { return this; }
@@ -394,7 +395,8 @@ public:
     {
         return createDiskDeserializer(ctx, activityId);
     }
-    virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) override{ }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) override {}
+    virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) override { }
     virtual IOutputMetaData * queryChildMeta(unsigned i) override { return NULL; }
 
     virtual const RtlRecord &queryRecordAccessor(bool expand) const override final;

rtl/include/eclhelper.hpp

@@ -545,7 +545,7 @@ interface IOutputMetaData : public IRecordSize
     virtual void toXML(const byte * self, IXmlWriter & out) = 0;
     virtual unsigned getVersion() const = 0;
     virtual unsigned getMetaFlags() = 0;
-    virtual const RtlTypeInfo * queryTypeInfo() const { return NULL; }                                          // non null for meta from generated code
+    virtual const RtlTypeInfo * queryTypeInfo() const = 0;
 
     virtual void destruct(byte * self) = 0;
 
@@ -557,7 +557,7 @@ interface IOutputMetaData : public IRecordSize
     virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) = 0;        // ctx is currently allowed to be NULL
     virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) = 0;
 
-    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}            // from and to are *hints* for the range of fields to call through with
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) = 0;           // from and to are *hints* for the range of fields to call through with
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) = 0;
     virtual IOutputMetaData * queryChildMeta(unsigned i) = 0;
     virtual const RtlRecord &queryRecordAccessor(bool expand) const = 0;
@@ -1210,7 +1210,7 @@ struct IHThorIndexWriteArg : public IHThorArg
 struct IHThorFirstNArg : public IHThorArg
 {
     virtual __int64 getLimit() = 0;
-    virtual __int64 numToSkip()                             { return 0; }
+    virtual __int64 numToSkip() = 0;
     virtual bool preserveGrouping() = 0;
 };
 
@@ -1247,13 +1247,13 @@ struct IHThorDiskWriteArg : public IHThorArg
 struct IHThorFilterArg : public IHThorArg
 {
     virtual bool isValid(const void * _left) = 0;
-    virtual bool canMatchAny()                              { return true; }
+    virtual bool canMatchAny() = 0;
 };
 
 struct IHThorFilterGroupArg : public IHThorArg
 {
     virtual bool isValid(unsigned _num, const void * * _rows) = 0;
-    virtual bool canMatchAny()                              { return true; }
+    virtual bool canMatchAny() = 0;
 };
 
 struct IHThorGroupArg : public IHThorArg
@@ -1286,13 +1286,13 @@ struct IHThorProcessArg : public IHThorArg
 
 struct IHThorProjectArg : public IHThorArg
 {
-    virtual bool canFilter()                                { return false; }
+    virtual bool canFilter() = 0;
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _left) = 0;
 };
 
 struct IHThorCountProjectArg : public IHThorArg
 {
-    virtual bool canFilter()                                { return false; }
+    virtual bool canFilter() = 0;
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _left, unsigned __int64 _counter) = 0;
 };
 
@@ -1337,19 +1337,19 @@ struct IHThorQuantileArg : public IHThorArg
 
 struct IHThorCombineArg : public IHThorArg
 {
-    virtual bool canFilter()                                { return false; }
+    virtual bool canFilter() = 0;
     virtual size32_t transform(ARowBuilder & rowBuilder, unsigned _num, const void * * _rows) = 0;
 };
 
 struct IHThorCombineGroupArg : public IHThorArg
 {
-    virtual bool canFilter()                                { return false; }
+    virtual bool canFilter() = 0;
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _left, unsigned _num, const void * * _rows) = 0;
 };
 
 struct IHThorRollupGroupArg : public IHThorArg
 {
-    virtual bool canFilter()                                { return false; }
+    virtual bool canFilter() = 0;
     virtual size32_t transform(ARowBuilder & rowBuilder, unsigned _num, const void * * _rows) = 0;
 };
 
@@ -1360,7 +1360,7 @@ typedef IHThorArg IHThorNullArg;
 
 struct IHThorActionArg : public IHThorArg
 {
-    virtual void action() {};
+    virtual void action() = 0;
 };
 typedef IHThorActionArg IHThorSideEffectArg;
 
@@ -1390,7 +1390,7 @@ struct IHThorCatchArg : public IHThorArg
 struct IHThorSplitArg : public IHThorArg
 {
     virtual unsigned numBranches() = 0;
-    virtual bool isBalanced()                               { return false; }
+    virtual bool isBalanced() = 0;
 };
 
 struct IHThorSpillExtra : public IInterface
@@ -1741,10 +1741,10 @@ struct IHThorFetchContext
     virtual const char * getFileName() = 0;                 // Returns filename of raw file fpos'es refer into
     virtual IOutputMetaData * queryDiskRecordSize() = 0;            // Expected layout
     virtual IOutputMetaData * queryProjectedDiskRecordSize() = 0;   // Projected layout
-    virtual unsigned getFetchFlags() { return 0; }              
-    virtual unsigned getDiskFormatCrc() { return 0; }
-    virtual unsigned getProjectedFormatCrc() { return getDiskFormatCrc(); }  // Should really be crc of queryProjectedDiskRecordSize layout
-    virtual void getFileEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
+    virtual unsigned getFetchFlags() = 0;
+    virtual unsigned getDiskFormatCrc() = 0;
+    virtual unsigned getProjectedFormatCrc() = 0;
+    virtual void getFileEncryptKey(size32_t & keyLen, void * & key) = 0;
 };
 
 struct IHThorKeyedJoinBaseArg : public IHThorArg
@@ -1752,7 +1752,7 @@ struct IHThorKeyedJoinBaseArg : public IHThorArg
     // For the data going to the indexRead remote activity:
     virtual size32_t extractIndexReadFields(ARowBuilder & rowBuilder, const void * _input) = 0;
     virtual IOutputMetaData * queryIndexReadInputRecordSize() = 0;
-    virtual bool leftCanMatch(const void * inputRow) { return true; }
+    virtual bool leftCanMatch(const void * inputRow) = 0;
 
     // Inside the indexRead remote activity:
     virtual const char * getIndexFileName() = 0;
@@ -1763,7 +1763,7 @@ struct IHThorKeyedJoinBaseArg : public IHThorArg
     virtual unsigned getJoinLimit() = 0;                                        // if a key joins more than this limit no records are output (0 = no limit)
     virtual unsigned getKeepLimit() = 0;                                        // limit to number of matches that are kept (0 = no limit)
     virtual unsigned getIndexFormatCrc() = 0;
-    virtual unsigned getProjectedIndexFormatCrc() { return getIndexFormatCrc(); }  // Should really be crc of queryIndexReadInputRecordSize layout (?)
+    virtual unsigned getProjectedIndexFormatCrc() = 0;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) = 0;
 
     // For the data going to the fetch remote activity:
@@ -1779,20 +1779,20 @@ struct IHThorKeyedJoinBaseArg : public IHThorArg
     virtual size32_t createDefaultRight(ARowBuilder & rowBuilder) = 0;
     virtual unsigned getJoinFlags() = 0;
     virtual bool diskAccessRequired() = 0;                  // if false, all transform values can be fulfilled from the key, which is passed as right.
-    virtual unsigned __int64 getRowLimit()                  { return (unsigned __int64) -1; }
-    virtual void onLimitExceeded()                          { }
-    virtual unsigned __int64 getSkipLimit()                 { return 0; }
+    virtual unsigned __int64 getRowLimit() = 0;
+    virtual void onLimitExceeded() = 0;
+    virtual unsigned __int64 getSkipLimit() = 0;
     virtual unsigned getMatchAbortLimit() = 0;
-    virtual void onMatchAbortLimitExceeded()                { }
+    virtual void onMatchAbortLimitExceeded() = 0;
 
     //keyedFpos is always 0
-    virtual size32_t onFailTransform(ARowBuilder & rowBuilder, const void * _dummyRight, const void * _origRow, unsigned __int64 keyedFpos, IException * e) { return 0; }
+    virtual size32_t onFailTransform(ARowBuilder & rowBuilder, const void * _dummyRight, const void * _origRow, unsigned __int64 keyedFpos, IException * e) = 0;
 //Join:
 //Denormalize:
     //The _filepos field is used for full keyed join to pass in the fileposition.  It should really use the disk IThorDiskCallback interface.
-    virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned __int64 keyedFpos, unsigned counter) { return 0; }
+    virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned __int64 keyedFpos, unsigned counter) = 0;
 //Denormalize group:
-    virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned _numRows, const void * * _rows) { return 0; }
+    virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned _numRows, const void * * _rows) = 0;
 
     inline bool hasNewSegmentMonitors()                     { return (getJoinFlags() & JFnewfilters) != 0; }
 
@@ -1848,12 +1848,12 @@ struct IHThorKeyedDistributeArg : public IHThorArg
 
 struct IHThorFetchBaseArg : public IHThorArg, public IHThorFetchContext
 {
-    virtual unsigned __int64 getRowLimit() { return (unsigned __int64) -1; }
-    virtual void onLimitExceeded()         { }
-    inline  bool transformNeedsRhs()       { return queryExtractedSize() != 0; }
-    virtual size32_t extractJoinFields(ARowBuilder & rowBuilder, const void * _right) { return 0; }
-    virtual bool extractAllJoinFields()    { return false; }
-    virtual IOutputMetaData * queryExtractedSize() { return NULL; }               
+    virtual unsigned __int64 getRowLimit() = 0;
+    virtual void onLimitExceeded() = 0;
+    inline  bool transformNeedsRhs()       { return queryExtractedSize() != nullptr; }
+    virtual size32_t extractJoinFields(ARowBuilder & rowBuilder, const void * _right) = 0;
+    virtual bool extractAllJoinFields() = 0;
+    virtual IOutputMetaData * queryExtractedSize() = 0;
 };
 
 struct IHThorBinFetchExtra : public IInterface
@@ -2003,7 +2003,7 @@ struct IHThorLocalResultReadArg : public IHThorArg
 struct IHThorLocalResultWriteArg : public IHThorArg
 {
     virtual unsigned querySequence() = 0;
-    virtual bool usedOutsideGraph() { return true; }
+    virtual bool usedOutsideGraph() = 0;
 };
 
 struct IHThorGraphLoopResultReadArg : public IHThorArg
@@ -2088,13 +2088,13 @@ struct IHThorXmlParseArg : public IHThorArg
     virtual const char * getXmlIteratorPath() = 0;
     virtual void getSearchText(size32_t & retLen, char * & retText, const void * _self) = 0;
     virtual bool searchTextNeedsFree() = 0;
-    virtual bool requiresContents() { return false; }
+    virtual bool requiresContents() = 0;
 };
 
 struct IHThorXmlFetchExtra : public IInterface
 {
     virtual size32_t transform(ARowBuilder & rowBuilder, IColumnProvider * rowLeft, const void * right, unsigned __int64 _fpos) = 0;
-    virtual bool requiresContents() { return false; }
+    virtual bool requiresContents() = 0;
 };
 
 struct IHThorXmlFetchArg : public IHThorFetchBaseArg, public IHThorXmlFetchExtra
@@ -2106,10 +2106,10 @@ struct IHThorXmlFetchArg : public IHThorFetchBaseArg, public IHThorXmlFetchExtra
 struct IHThorXmlWriteExtra : public IInterface
 {
     virtual void toXML(const byte * self, IXmlWriter & out) = 0;
-    virtual const char * getXmlIteratorPath()         { return NULL; }             // supplies the prefix and suffix for a row
-    virtual const char * getHeader()                   { return NULL; }
-    virtual const char * getFooter()                   { return NULL; }
-    virtual unsigned getXmlFlags()                     { return 0; }
+    virtual const char * getXmlIteratorPath() = 0;
+    virtual const char * getHeader() = 0;
+    virtual const char * getFooter() = 0;
+    virtual unsigned getXmlFlags() = 0;
 };
 
 struct IHThorXmlWriteArg : public IHThorDiskWriteArg, public IHThorXmlWriteExtra
@@ -2151,8 +2151,8 @@ struct IHThorPipeWriteArg : public IHThorArg
     virtual const char * getPipeProgram() = 0;
     virtual int getSequence() = 0;
     virtual IOutputMetaData * queryDiskRecordSize() = 0;
-    virtual char * getNameFromRow(const void * _self)       { return NULL; }
-    virtual bool recreateEachRow()                          { return false; }
+    virtual char * getNameFromRow(const void * _self) = 0;
+    virtual bool recreateEachRow() = 0;
     virtual unsigned getPipeFlags() = 0;
     virtual IHThorCsvWriteExtra * queryCsvOutput() = 0;
     virtual IHThorXmlWriteExtra * queryXmlOutput() = 0;
@@ -2161,8 +2161,8 @@ struct IHThorPipeWriteArg : public IHThorArg
 struct IHThorPipeThroughArg : public IHThorArg
 {
     virtual const char * getPipeProgram() = 0;
-    virtual char * getNameFromRow(const void * _self)       { return NULL; }
-    virtual bool recreateEachRow()                          { return false; }
+    virtual char * getNameFromRow(const void * _self) = 0;
+    virtual bool recreateEachRow() = 0;
     virtual unsigned getPipeFlags() = 0;
     virtual IHThorCsvWriteExtra * queryCsvOutput() = 0;
     virtual IHThorXmlWriteExtra * queryXmlOutput() = 0;
@@ -2347,10 +2347,10 @@ interface IHThorSteppedSourceExtra
 // where-ever possible because that improves the scope for cse.
 struct IHThorCompoundBaseArg : public IHThorArg
 {
-    virtual bool canMatchAny()                              { return true; }
-    virtual void createSegmentMonitors(IIndexReadContext *ctx) {}
-    virtual bool canMatch(const void * row)                 { return true; }
-    virtual bool hasMatchFilter()                           { return false; }
+    virtual bool canMatchAny() = 0;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) = 0;
+    virtual bool canMatch(const void * row) = 0;
+    virtual bool hasMatchFilter() = 0;
 };
 
 struct IHThorIndexReadBaseArg : extends IHThorCompoundBaseArg
@@ -2366,7 +2366,7 @@ struct IHThorIndexReadBaseArg : extends IHThorCompoundBaseArg
 
     inline bool hasSegmentMonitors()                        { return (getFlags() & TIRnofilter) == 0; }
     inline bool hasNewSegmentMonitors()                     { return (getFlags() & TIRnewfilters) != 0; }
-    virtual IHThorSteppedSourceExtra *querySteppingExtra()  { return NULL; }
+    virtual IHThorSteppedSourceExtra *querySteppingExtra() = 0;
 };
 
 struct IHThorDiskReadBaseArg : extends IHThorCompoundBaseArg
@@ -2377,7 +2377,7 @@ struct IHThorDiskReadBaseArg : extends IHThorCompoundBaseArg
     virtual unsigned getFlags() = 0;
     virtual unsigned getDiskFormatCrc() = 0;
     virtual unsigned getProjectedFormatCrc() = 0;
-    virtual void getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) = 0;
     virtual void setCallback(IThorDiskCallback * callback) = 0;
 
     inline bool hasSegmentMonitors()                        { return (getFlags() & TDRkeyed) != 0; }
@@ -2388,9 +2388,9 @@ struct IHThorDiskReadBaseArg : extends IHThorCompoundBaseArg
 // common between Read, Normalize
 struct IHThorCompoundExtra : public IInterface
 {
-    virtual unsigned __int64 getChooseNLimit()              { return I64C(0x7fffffffffffffff); }
-    virtual unsigned __int64 getRowLimit()                  { return (unsigned __int64) -1; }
-    virtual void onLimitExceeded()                          { }
+    virtual unsigned __int64 getChooseNLimit() = 0;
+    virtual unsigned __int64 getRowLimit() = 0;
+    virtual void onLimitExceeded() = 0;
 };
 
 struct IHThorSourceCountLimit : public IInterface
@@ -2428,8 +2428,8 @@ struct IHThorCompoundNormalizeExtra : public IHThorCompoundExtra
     virtual bool first(const void * src) = 0;
     virtual bool next() = 0;                            //NB: src from transformFirst() must stay in memory while transformNext() is being called.
     virtual size32_t transform(ARowBuilder & rowBuilder) = 0;
-    virtual unsigned __int64 getKeyedLimit()                { return (unsigned __int64) -1; }
-    virtual void onKeyedLimitExceeded()                     { }
+    virtual unsigned __int64 getKeyedLimit() = 0;
+    virtual void onKeyedLimitExceeded() = 0;
 };
 
 
@@ -2446,10 +2446,10 @@ struct IHThorCompoundAggregateExtra : public IInterface
 //Count
 struct IHThorCompoundCountExtra : public IInterface
 {
-    virtual bool hasFilter()                                                { return false; }       // also true if denormalized(!)
-    virtual size32_t numValid(const void * src)                             { return 1; }           //NB: Can be > 1 if source is normlized
+    virtual bool hasFilter() = 0;
+    virtual size32_t numValid(const void * src) = 0;
     virtual size32_t numValid(size32_t srcLen, const void * src) = 0;
-    virtual unsigned __int64 getChooseNLimit()                              { return (unsigned __int64) -1; }
+    virtual unsigned __int64 getChooseNLimit() = 0;
 };
 
 //NormalizeAggregate
@@ -2603,7 +2603,6 @@ struct IHThorChildNormalizeArg : public IHThorArg
     virtual bool first() = 0;
     virtual bool next() = 0;
     virtual size32_t transform(ARowBuilder & rowBuilder) = 0;
-//??    virtual bool canMatchAny()                              { return true; }
 };
 
 //Aggregate

rtl/include/eclhelper_base.hpp

@@ -108,6 +108,15 @@ public:
     virtual const char * getXmlIteratorPath() override;
 };
 
+//A class which only implements default values for the interface
+class ECLRTL_API CHThorXmlWriteExtra : public IHThorXmlWriteExtra
+{
+    virtual const char * getXmlIteratorPath() override;
+    virtual const char * getHeader() override;
+    virtual const char * getFooter() override;
+    virtual unsigned getXmlFlags() override;
+};
+
 class ECLRTL_API CThorPipeWriteArg : public CThorSinkArgOf<IHThorPipeWriteArg>
 {
 public:
@@ -219,6 +228,7 @@ class ECLRTL_API CThorCombineGroupArg : public CThorArgOf<IHThorCombineGroupArg>
 
 class ECLRTL_API CThorRollupGroupArg : public CThorArgOf<IHThorRollupGroupArg>
 {
+    virtual bool canFilter() override;
 };
 
 class ECLRTL_API CThorRegroupArg : public CThorArgOf<IHThorRegroupArg>
@@ -462,9 +472,15 @@ class ECLRTL_API CThorKeyedJoinArg : public CThorArgOf<IHThorKeyedJoinArg>
     virtual unsigned getMatchAbortLimit() override;
     virtual void onMatchAbortLimitExceeded() override;
 
+    virtual unsigned getFetchFlags() override;
+    virtual unsigned getDiskFormatCrc() override;
+    virtual unsigned getProjectedFormatCrc() override;
+    virtual void getFileEncryptKey(size32_t & keyLen, void * & key) override;
+
     virtual unsigned getJoinLimit() override;
     virtual unsigned getKeepLimit() override;
     virtual unsigned getJoinFlags() override;
+    virtual unsigned getProjectedIndexFormatCrc() override;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) override;
 
     virtual size32_t extractFetchFields(ARowBuilder & rowBuilder, const void * _input) override;
@@ -585,6 +601,15 @@ class ECLRTL_API CThorKeyedDistributeArg : public CThorArgOf<IHThorKeyedDistribu
 
 class ECLRTL_API CThorFetchArg : public CThorArgOf<IHThorFetchArg>
 {
+    virtual unsigned getFetchFlags() override;
+    virtual unsigned getDiskFormatCrc() override;
+    virtual unsigned getProjectedFormatCrc() override;
+    virtual void getFileEncryptKey(size32_t & keyLen, void * & key) override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
+    virtual size32_t extractJoinFields(ARowBuilder & rowBuilder, const void * _right) override;
+    virtual bool extractAllJoinFields() override;
+    virtual IOutputMetaData * queryExtractedSize() override;
 };
 
 class ECLRTL_API CThorWorkUnitWriteArg : public CThorSinkArgOf<IHThorWorkUnitWriteArg>
@@ -711,6 +736,15 @@ class ECLRTL_API CThorCsvWriteArg : public CThorSinkArgOf<IHThorCsvWriteArg>
 
 class ECLRTL_API CThorCsvFetchArg: public CThorArgOf<IHThorCsvFetchArg>
 {
+    virtual unsigned getFetchFlags() override;
+    virtual unsigned getDiskFormatCrc() override;
+    virtual unsigned getProjectedFormatCrc() override;
+    virtual void getFileEncryptKey(size32_t & keyLen, void * & key) override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
+    virtual size32_t extractJoinFields(ARowBuilder & rowBuilder, const void * _right) override;
+    virtual bool extractAllJoinFields() override;
+    virtual IOutputMetaData * queryExtractedSize() override;
 };
 
 //-- Xml --
@@ -722,7 +756,16 @@ class ECLRTL_API CThorXmlParseArg : public CThorArgOf<IHThorXmlParseArg>
 
 class ECLRTL_API CThorXmlFetchArg : public CThorArgOf<IHThorXmlFetchArg>
 {
+    virtual unsigned getFetchFlags() override;
+    virtual unsigned getDiskFormatCrc() override;
+    virtual unsigned getProjectedFormatCrc() override;
+    virtual void getFileEncryptKey(size32_t & keyLen, void * & key) override;
     virtual bool requiresContents() override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
+    virtual size32_t extractJoinFields(ARowBuilder & rowBuilder, const void * _right) override;
+    virtual bool extractAllJoinFields() override;
+    virtual IOutputMetaData * queryExtractedSize() override;
 };
 
 //Simple xml generation...
@@ -814,6 +857,15 @@ class ECLRTL_API CThorIndexReadArg : public CThorArgOf<IHThorIndexReadArg>
     virtual unsigned getFlags() override;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) override;
     virtual void setCallback(IThorIndexCallback * _tc) override;
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual IHThorSteppedSourceExtra *querySteppingExtra() override;
+
+    virtual unsigned __int64 getChooseNLimit() override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
 
     virtual bool needTransform() override;
     virtual bool transformMayFilter() override;
@@ -826,7 +878,6 @@ class ECLRTL_API CThorIndexReadArg : public CThorArgOf<IHThorIndexReadArg>
 
     virtual size32_t transformOnLimitExceeded(ARowBuilder & rowBuilder) override;
     virtual size32_t transformOnKeyedLimitExceeded(ARowBuilder & rowBuilder) override;
-    virtual IHThorSteppedSourceExtra *querySteppingExtra() override;
 
 public:
     IThorIndexCallback * fpp;
@@ -845,6 +896,17 @@ class ECLRTL_API CThorIndexNormalizeArg : public CThorArgOf<IHThorIndexNormalize
     virtual unsigned getFlags() override;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) override;
     virtual void setCallback(IThorIndexCallback * _tc) override;
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual IHThorSteppedSourceExtra *querySteppingExtra() override;
+
+    virtual unsigned __int64 getChooseNLimit() override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
+    virtual unsigned __int64 getKeyedLimit() override;
+    virtual void onKeyedLimitExceeded() override;
 
     virtual size32_t transformOnLimitExceeded(ARowBuilder & rowBuilder) override;
     virtual size32_t transformOnKeyedLimitExceeded(ARowBuilder & rowBuilder) override;
@@ -858,6 +920,11 @@ class ECLRTL_API CThorIndexAggregateArg : public CThorArgOf<IHThorIndexAggregate
     virtual unsigned getFlags() override;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) override;
     virtual void setCallback(IThorIndexCallback * _tc) override;
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual IHThorSteppedSourceExtra *querySteppingExtra() override;
     virtual size32_t mergeAggregate(ARowBuilder & rowBuilder, const void * src) override;
     virtual void processRows(ARowBuilder & rowBuilder, size32_t srcLen, const void * src) override;
 
@@ -870,13 +937,21 @@ class ECLRTL_API CThorIndexCountArg : public CThorArgOf<IHThorIndexCountArg>
     virtual unsigned getFlags() override;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) override;
     virtual void setCallback(IThorIndexCallback * _tc) override;
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual IHThorSteppedSourceExtra *querySteppingExtra() override;
 
     virtual unsigned __int64 getRowLimit() override;
     virtual void onLimitExceeded() override;
     virtual unsigned __int64 getKeyedLimit() override;
     virtual void onKeyedLimitExceeded() override;
 
+    virtual bool hasFilter() override;
+    virtual size32_t numValid(const void * src) override;
     virtual size32_t numValid(size32_t srcLen, const void * _src) override;
+    virtual unsigned __int64 getChooseNLimit() override;
 
 public:
     IThorIndexCallback * fpp;
@@ -887,6 +962,11 @@ class ECLRTL_API CThorIndexGroupAggregateArg : public CThorArgOf<IHThorIndexGrou
     virtual unsigned getFlags() override;
     virtual bool getIndexLayout(size32_t & _retLen, void * & _retData) override;
     virtual void setCallback(IThorIndexCallback * _tc) override;
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual IHThorSteppedSourceExtra *querySteppingExtra() override;
     virtual bool createGroupSegmentMonitors(IIndexReadContext *ctx) override;
     virtual unsigned getGroupingMaxField() override;
     virtual size32_t initialiseCountGrouping(ARowBuilder & rowBuilder, const void * src) override;
@@ -904,6 +984,16 @@ class ECLRTL_API CThorDiskReadArg : public CThorArgOf<IHThorDiskReadArg>
     virtual unsigned getFlags() override;
     virtual void setCallback(IThorDiskCallback * _tc) override;
 
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) override;
+
+    virtual unsigned __int64 getChooseNLimit() override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
+
     virtual bool needTransform() override;
     virtual bool transformMayFilter() override;
     virtual unsigned __int64 getKeyedLimit() override;
@@ -926,6 +1016,18 @@ class ECLRTL_API CThorDiskNormalizeArg : public CThorArgOf<IHThorDiskNormalizeAr
     virtual unsigned getFlags() override;
     virtual void setCallback(IThorDiskCallback * _tc) override;
 
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) override;
+
+    virtual unsigned __int64 getChooseNLimit() override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
+    virtual unsigned __int64 getKeyedLimit() override;
+    virtual void onKeyedLimitExceeded() override;
+
     virtual size32_t transformOnLimitExceeded(ARowBuilder & rowBuilder) override;
     virtual size32_t transformOnKeyedLimitExceeded(ARowBuilder & rowBuilder) override;
 
@@ -937,6 +1039,11 @@ class ECLRTL_API CThorDiskAggregateArg : public CThorArgOf<IHThorDiskAggregateAr
 {
     virtual unsigned getFlags() override;
     virtual void setCallback(IThorDiskCallback * _tc) override;
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) override;
     virtual size32_t mergeAggregate(ARowBuilder & rowBuilder, const void * src) override;
 
 public:
@@ -947,12 +1054,20 @@ class ECLRTL_API CThorDiskCountArg : public CThorArgOf<IHThorDiskCountArg>
 {
     virtual unsigned getFlags() override;
     virtual void setCallback(IThorDiskCallback * _tc) override;
-
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) override;
     virtual unsigned __int64 getRowLimit() override;
     virtual void onLimitExceeded() override;
     virtual unsigned __int64 getKeyedLimit() override;
     virtual void onKeyedLimitExceeded() override;
 
+    virtual bool hasFilter() override;
+    virtual size32_t numValid(const void * src) override;
+    virtual unsigned __int64 getChooseNLimit() override;
+
 public:
     IThorDiskCallback * fpp;
 };
@@ -961,6 +1076,11 @@ class ECLRTL_API CThorDiskGroupAggregateArg : public CThorArgOf<IHThorDiskGroupA
 {
     virtual unsigned getFlags() override;
     virtual void setCallback(IThorDiskCallback * _tc) override;
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) override;
     virtual bool createGroupSegmentMonitors(IIndexReadContext *ctx) override;
     virtual unsigned getGroupingMaxField() override;
     virtual size32_t initialiseCountGrouping(ARowBuilder & rowBuilder, const void * src) override;
@@ -980,7 +1100,11 @@ class ECLRTL_API CThorCsvReadArg: public CThorArgOf<IHThorCsvReadArg>
     virtual unsigned getDiskFormatCrc() override;   // no meaning
     virtual unsigned getProjectedFormatCrc() override;   // no meaning
     virtual void setCallback(IThorDiskCallback * _tc) override;
-
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) override;
 public:
     IThorDiskCallback * fpp;
 };
@@ -994,7 +1118,11 @@ class ECLRTL_API CThorXmlReadArg: public CThorArgOf<IHThorXmlReadArg>
     virtual unsigned getDiskFormatCrc() override;   // no meaning
     virtual unsigned getProjectedFormatCrc() override;   // no meaning
     virtual void setCallback(IThorDiskCallback * _tc) override;
-
+    virtual bool canMatchAny() override;
+    virtual void createSegmentMonitors(IIndexReadContext *ctx) override;
+    virtual bool canMatch(const void * row) override;
+    virtual bool hasMatchFilter() override;
+    virtual void getEncryptKey(size32_t & keyLen, void * & key) override;
 public:
     IThorDiskCallback * fpp;
 };
@@ -1019,6 +1147,11 @@ class ECLRTL_API CThorChildGroupAggregateArg : public CThorArgOf<IHThorChildGrou
 //Normalize
 class ECLRTL_API CThorChildThroughNormalizeArg : public CThorArgOf<IHThorChildThroughNormalizeArg>
 {
+    virtual unsigned __int64 getChooseNLimit() override;
+    virtual unsigned __int64 getRowLimit() override;
+    virtual void onLimitExceeded() override;
+    virtual unsigned __int64 getKeyedLimit() override;
+    virtual void onKeyedLimitExceeded() override;
 };
 
 class ECLRTL_API CThorLoopArg : public CThorArgOf<IHThorLoopArg>

thorlcr/thorutil/thmem.cpp

@@ -2577,6 +2577,7 @@ public:
     virtual size32_t getRecordSize(const void *) { return extraSz + sizeof(const void *); }
     virtual size32_t getMinRecordSize() const { return extraSz + sizeof(const void *); }
     virtual size32_t getFixedSize() const { return extraSz + sizeof(const void *); }
+    virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
     virtual void toXML(const byte * self, IXmlWriter & out) 
     { 
          // ignoring xml'ing extra
@@ -2622,6 +2623,7 @@ public:
             internalDeserializer.setown(new CDeserializer(childMeta->createInternalDeserializer(ctx, activityId), childAllocator, extraSz));
         return LINK(internalDeserializer);
     }
+    virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
     virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) 
     {
         //GH: I think this is what it should do, please check