Merge pull request #10484 from richardkchapman/new-index-metadata

HPCC-18422 Add new format meta data to indexes

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/deftype/deftype.cpp

@@ -2195,6 +2195,17 @@ bool isIntegralType(ITypeInfo * type)
     return false;
 }
 
+bool isSimpleIntegralType(ITypeInfo * type)
+{
+    switch (type->getTypeCode())
+    {
+    case type_int:
+    case type_swapint:
+        return true;
+    }
+    return false;
+}
+
 bool isPatternType(ITypeInfo * type)
 {
     switch(type->getTypeCode())

common/deftype/deftype.hpp

@@ -242,6 +242,7 @@ extern DEFTYPE_API ITypeInfo * getPromotedCompareType(ITypeInfo * left, ITypeInf
 extern DEFTYPE_API bool isNumericType(ITypeInfo * type);
 extern DEFTYPE_API bool isStringType(ITypeInfo * type);
 extern DEFTYPE_API bool isSimpleStringType(ITypeInfo * type);
+extern DEFTYPE_API bool isSimpleIntegralType(ITypeInfo * type);
 extern DEFTYPE_API bool isIntegralType(ITypeInfo * type);
 extern DEFTYPE_API bool isPatternType(ITypeInfo * type);
 extern DEFTYPE_API bool isUnicodeType(ITypeInfo * type);

common/remote/sockfile.cpp

@@ -3038,8 +3038,20 @@ public:
     {
         if (!remoteSupport())
             return directKM->queryRecordSize();
+        return currentSize; // this is wrong I think
+    }
+    virtual size32_t queryRowSize() override
+    {
+        if (!remoteSupport())
+            return directKM->queryRowSize();
         return currentSize;
     }
+    virtual unsigned __int64 querySequence() override
+    {
+        if (!remoteSupport())
+            return directKM->querySequence();
+        UNIMPLEMENTED;
+    }
     virtual bool lookup(bool exact) override
     {
         if (!remoteSupport())

dali/base/dacsds.ipp

@@ -296,7 +296,7 @@ class CClientRemoteTree : implements ITrackChanges, public CRemoteTreeBase
 public:
     CClientRemoteTree(CRemoteConnection &conn, CPState _state=CPS_Unchanged);
     CClientRemoteTree(const char *name, IPTArrayValue *value, ChildMap *children, CRemoteConnection &conn, CPState _state=CPS_Unchanged);
-    void beforeDispose();
+    virtual void beforeDispose() override;
     void resetState(unsigned state, bool sub=false);
     inline bool queryStateChanges() const { return serverId && connection.queryStateChanges(); }
     inline unsigned queryState() { return state; }
@@ -306,8 +306,8 @@ public:
     IPropertyTree *collateData();
     void clearCommitChanges(MemoryBuffer *mb=NULL);
 
-    virtual void Link() const;
-    virtual bool Release() const;
+    virtual void Link() const override;
+    virtual bool Release() const override;
 
     virtual void deserializeSelfRT(MemoryBuffer &mb) override;
     virtual void deserializeChildrenRT(MemoryBuffer &src) override;

dali/base/dadfs.cpp

@@ -3017,6 +3017,7 @@ public:
                     queryLogicalName(), (superBlocked?"blocked":"unblocked"));
 
 #ifdef SUBFILE_COMPATIBILITY_CHECKING
+        // MORE - this first check looks completely useless to me
         bool subSoft = subProp.hasProp("_record_layout");
         bool superSoft = superProp.hasProp("_record_layout");
         if (superSoft != subSoft)
@@ -3202,7 +3203,7 @@ protected:
     StringAttr partmask;
     FileClusterInfoArray clusters;
 
-    void savePartsAttr(bool force)
+    void savePartsAttr(bool force) override
     {
         CriticalBlock block (sect);
         IPropertyTree *pt;
@@ -3411,7 +3412,7 @@ protected: friend class CDistributedFilePart;
         return parts;
     }
 public:
-    IMPLEMENT_IINTERFACE;
+    IMPLEMENT_IINTERFACE_O;
 
     CDistributedFile(CDistributedFileDirectory *_parent, IRemoteConnection *_conn,const CDfsLogicalFileName &lname,IUserDescriptor *user) // takes ownership of conn
     {
@@ -3513,7 +3514,7 @@ public:
         clusters.kill();
     }
 
-    IFileDescriptor *getFileDescriptor(const char *_clusterName)
+    IFileDescriptor *getFileDescriptor(const char *_clusterName) override
     {
         CriticalBlock block (sect);
         Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(root,&queryNamedGroupStore(),0);
@@ -3588,17 +3589,17 @@ public:
             ERRLOG("No cluster specified for %s",logicalName.get());
     }
 
-    unsigned numClusters()
+    virtual unsigned numClusters() override
     {
         return clusters.ordinality();
     }
 
-    unsigned findCluster(const char *clustername)
+    virtual unsigned findCluster(const char *clustername) override
     {
         return clusters.find(clustername);
     }
 
-    unsigned getClusterNames(StringArray &clusternames)
+    virtual unsigned getClusterNames(StringArray &clusternames) override
     {
         return clusters.getNames(clusternames);
     }
@@ -3668,7 +3669,7 @@ public:
             CDistributedFileBase<IDistributedFile>::conn->commit(); // should only be cluster changes but a bit dangerous
     }
 
-    void addCluster(const char *clustername,const ClusterPartDiskMapSpec &mspec)
+    virtual void addCluster(const char *clustername,const ClusterPartDiskMapSpec &mspec) override
     {
         if (!clustername&&!*clustername)
             return;
@@ -3689,7 +3690,7 @@ public:
         saveClusters();
     }
 
-    bool removeCluster(const char *clustername)
+    virtual bool removeCluster(const char *clustername) override
     {
         CClustersLockedSection cls(CDistributedFileBase<IDistributedFile>::logicalName, true);
         reloadClusters();
@@ -3717,7 +3718,7 @@ public:
         return false;
     }
 
-    void setPreferredClusters(const char *clusterlist)
+    virtual void setPreferredClusters(const char *clusterlist) override
     {
         clusters.setPreferred(clusterlist,CDistributedFileBase<IDistributedFile>::logicalName);
     }
@@ -3759,7 +3760,7 @@ public:
     }
 
 
-    StringBuffer &getClusterName(unsigned clusternum,StringBuffer &name)
+    virtual StringBuffer &getClusterName(unsigned clusternum,StringBuffer &name) override
     {
         return clusters.getName(clusternum,name);
     }
@@ -3769,13 +3770,13 @@ public:
         return clusters.copyNum(part,copy, numParts(),replicate);
     }
 
-    ClusterPartDiskMapSpec &queryPartDiskMapping(unsigned clusternum)
+    virtual ClusterPartDiskMapSpec &queryPartDiskMapping(unsigned clusternum) override
     {
         assertex(clusternum<clusters.ordinality());
         return clusters.queryPartDiskMapping(clusternum);
     }
 
-    void updatePartDiskMapping(const char *clustername,const ClusterPartDiskMapSpec &spec)
+    virtual void updatePartDiskMapping(const char *clustername,const ClusterPartDiskMapSpec &spec) override
     {
         CClustersLockedSection cls(CDistributedFileBase<IDistributedFile>::logicalName, true);
         reloadClusters();
@@ -3786,22 +3787,22 @@ public:
         }
     }
 
-    IGroup *queryClusterGroup(unsigned clusternum)
+    virtual IGroup *queryClusterGroup(unsigned clusternum) override
     {
         return clusters.queryGroup(clusternum);
     }
 
-    StringBuffer &getClusterGroupName(unsigned clusternum, StringBuffer &name)
+    virtual StringBuffer &getClusterGroupName(unsigned clusternum, StringBuffer &name) override
     {
         return clusters.item(clusternum).getGroupName(name, &queryNamedGroupStore());
     }
 
-    virtual unsigned numCopies(unsigned partno)
+    virtual unsigned numCopies(unsigned partno) override
     {
         return clusters.numCopies(partno,numParts());
     }
 
-    void setSingleClusterOnly()
+    virtual void setSingleClusterOnly() override
     {
         clusters.setSingleClusterOnly();
     }
@@ -3885,19 +3886,19 @@ public:
         }
     }
 
-    unsigned numParts()
+    virtual unsigned numParts() override
     {
         return parts.ordinality();
     }
 
-    IDistributedFilePart &queryPart(unsigned idx)
+    virtual IDistributedFilePart &queryPart(unsigned idx) override
     {
         if (idx<parts.ordinality())
             return queryParts().item(idx);
         return *(IDistributedFilePart *)NULL;
     }
 
-    IDistributedFilePart* getPart(unsigned idx)
+    virtual IDistributedFilePart* getPart(unsigned idx) override
     {
         if (idx>=parts.ordinality())
             return NULL;
@@ -3905,12 +3906,12 @@ public:
         return LINK(ret);
     }
 
-    IDistributedFilePartIterator *getIterator(IDFPartFilter *filter=NULL)
+    virtual IDistributedFilePartIterator *getIterator(IDFPartFilter *filter=NULL) override
     {
         return new CDistributedFilePartIterator(queryParts(),filter);
     }
 
-    void rename(const char *_logicalname,IUserDescriptor *user)
+    virtual void rename(const char *_logicalname,IUserDescriptor *user) override
     {
         StringBuffer prevname;
         Owned<IFileRelationshipIterator> reliter;
@@ -3943,13 +3944,13 @@ public:
     }
 
 
-    const char *queryDefaultDir()
+    virtual const char *queryDefaultDir() override
     {
         CriticalBlock block (sect);
         return directory.get();
     }
 
-    const char *queryPartMask()
+    virtual const char *queryPartMask() override
     {
         CriticalBlock block (sect);
         if (partmask.isEmpty()) {
@@ -3964,7 +3965,7 @@ public:
         return (!logicalName.isSet());
     }
 
-    void attach(const char *_logicalname,IUserDescriptor *user)
+    virtual void attach(const char *_logicalname,IUserDescriptor *user) override
     {
         CriticalBlock block (sect);
         assertex(isAnon()); // already attached!
@@ -4010,12 +4011,12 @@ public:
     }
 
 public:
-    virtual void detach(unsigned timeoutMs=INFINITE, ICodeContext *ctx=NULL)
+    virtual void detach(unsigned timeoutMs=INFINITE, ICodeContext *ctx=NULL) override
     {
         detach(timeoutMs, true, ctx);
     }
 
-    bool existsPhysicalPartFiles(unsigned short port)
+    virtual bool existsPhysicalPartFiles(unsigned short port) override
     {
         unsigned width = numParts();
         CriticalSection errcrit;
@@ -4109,10 +4110,10 @@ public:
         return false;
     }
 
-    bool renamePhysicalPartFiles(const char *newname,
-                                 const char *cluster,
-                                 IMultiException *mexcept,
-                                 const char *newbasedir)
+    virtual bool renamePhysicalPartFiles(const char *newname,
+                                         const char *cluster,
+                                         IMultiException *mexcept,
+                                         const char *newbasedir) override
     {
         // cluster TBD
         unsigned width = numParts();
@@ -4404,7 +4405,7 @@ public:
 
     IPropertyTree *queryRoot() { return root; }
 
-    __int64 getFileSize(bool allowphysical,bool forcephysical)
+    virtual __int64 getFileSize(bool allowphysical,bool forcephysical) override
     {
         __int64 ret = (__int64)(forcephysical?-1:queryAttributes().getPropInt64("@size",-1));
         if (ret==-1)
@@ -4426,7 +4427,7 @@ public:
         return ret;
     }
 
-    __int64 getDiskSize(bool allowphysical,bool forcephysical)
+    virtual __int64 getDiskSize(bool allowphysical,bool forcephysical) override
     {
         if (!isCompressed(NULL))
             return getFileSize(allowphysical, forcephysical);
@@ -4451,7 +4452,7 @@ public:
         return ret;
     }
 
-    bool getFileCheckSum(unsigned &checkSum)
+    virtual bool getFileCheckSum(unsigned &checkSum) override
     {
         if (queryAttributes().hasProp("@checkSum"))
             checkSum = (unsigned)queryAttributes().getPropInt64("@checkSum");
@@ -4470,7 +4471,7 @@ public:
         return true;
     }
 
-    virtual bool getFormatCrc(unsigned &crc)
+    virtual bool getFormatCrc(unsigned &crc) override
     {
         if (queryAttributes().hasProp("@formatCrc")) {
             // NB pre record_layout CRCs are not valid
@@ -4480,12 +4481,12 @@ public:
         return false;
     }
 
-    virtual bool getRecordLayout(MemoryBuffer &layout)
+    virtual bool getRecordLayout(MemoryBuffer &layout, const char *attrname) override
     {
-        return queryAttributes().getPropBin("_record_layout",layout);
+        return queryAttributes().getPropBin(attrname, layout);
     }
 
-    virtual bool getRecordSize(size32_t &rsz)
+    virtual bool getRecordSize(size32_t &rsz) override
     {
         if (queryAttributes().hasProp("@recordSize")) {
             rsz = (size32_t)queryAttributes().getPropInt("@recordSize");
@@ -4494,7 +4495,7 @@ public:
         return false;
     }
 
-    virtual unsigned getPositionPart(offset_t pos, offset_t &base)
+    virtual unsigned getPositionPart(offset_t pos, offset_t &base) override
     {
         unsigned n = numParts();
         base = 0;
@@ -4511,12 +4512,12 @@ public:
         return NotFound;
     }
 
-    IDistributedSuperFile *querySuperFile()
+    IDistributedSuperFile *querySuperFile() override
     {
         return NULL; // i.e. this isn't super file
     }
 
-    virtual bool checkClusterCompatible(IFileDescriptor &fdesc, StringBuffer &err)
+    virtual bool checkClusterCompatible(IFileDescriptor &fdesc, StringBuffer &err) override
     {
         unsigned n = numParts();
         if (fdesc.numParts()!=n) {
@@ -4559,7 +4560,7 @@ public:
         return true;
     }
 
-    void enqueueReplicate()
+    virtual void enqueueReplicate() override
     {
         MemoryBuffer mb;
         mb.append((byte)DRQ_REPLICATE).append(queryLogicalName());
@@ -4572,7 +4573,7 @@ public:
         qchannel->put(mb);
     }
 
-    bool getAccessedTime(CDateTime &dt)
+    virtual bool getAccessedTime(CDateTime &dt) override
     {
         StringBuffer str;
         if (!root->getProp("@accessed",str))
@@ -4581,7 +4582,7 @@ public:
         return true;
     }
 
-    virtual void setAccessedTime(const CDateTime &dt)
+    virtual void setAccessedTime(const CDateTime &dt) override
     {
         if (logicalName.isForeign())
             parent->setFileAccessed(logicalName,udesc,dt);
@@ -4610,14 +4611,14 @@ public:
         }
     }
 
-    void setAccessed()
+    virtual void setAccessed() override
     {
         CDateTime dt;
         dt.setNow();
         setAccessedTime(dt);
     }
 
-    virtual void validate()
+    virtual void validate() override
     {
         if (!existsPhysicalPartFiles(0))
         {
@@ -5278,7 +5279,7 @@ protected:
 
 public:
 
-    void checkFormatAttr(IDistributedFile *sub, const char* exprefix="")
+    virtual void checkFormatAttr(IDistributedFile *sub, const char* exprefix="") override
     {
         IDistributedSuperFile *superSub = sub->querySuperFile();
         if (superSub && (0 == superSub->numSubFiles(true)))
@@ -5298,7 +5299,7 @@ public:
         return NotFound;
     }
 
-    IMPLEMENT_IINTERFACE;
+    IMPLEMENT_IINTERFACE_O;
 
     void commonInit(CDistributedFileDirectory *_parent, IPropertyTree *_root)
     {
@@ -5354,7 +5355,7 @@ public:
         subfiles.kill();
     }
 
-    StringBuffer &getClusterName(unsigned clusternum,StringBuffer &name)
+    virtual StringBuffer &getClusterName(unsigned clusternum,StringBuffer &name) override
     {
         // returns the cluster name if all the same
         CriticalBlock block (sect);
@@ -5376,7 +5377,7 @@ public:
         return name;
     }
 
-    IFileDescriptor *getFileDescriptor(const char *clustername)
+    virtual IFileDescriptor *getFileDescriptor(const char *clustername) override
     {
         CriticalBlock block (sect);
         if (subfiles.ordinality()==1)
@@ -5499,7 +5500,7 @@ public:
         return fdesc.getClear();
     }
 
-    unsigned numParts()
+    virtual unsigned numParts() override
     {
         CriticalBlock block(sect);
         unsigned ret=0;
@@ -5508,7 +5509,7 @@ public:
         return ret;
     }
 
-    IDistributedFilePart &queryPart(unsigned idx)
+    virtual IDistributedFilePart &queryPart(unsigned idx) override
     {
         CriticalBlock block(sect);
         if (subfiles.ordinality()==1)
@@ -5520,13 +5521,13 @@ public:
         return partscache.item(idx);
     }
 
-    IDistributedFilePart* getPart(unsigned idx)
+    virtual IDistributedFilePart* getPart(unsigned idx) override
     {
         IDistributedFilePart* ret = &queryPart(idx);
         return LINK(ret);
     }
 
-    IDistributedFilePartIterator *getIterator(IDFPartFilter *filter=NULL)
+    virtual IDistributedFilePartIterator *getIterator(IDFPartFilter *filter=NULL) override
     {
         CriticalBlock block(sect);
         if (subfiles.ordinality()==1)
@@ -5536,7 +5537,7 @@ public:
         return ret;
     }
 
-    void rename(const char *_logicalname,IUserDescriptor *user)
+    virtual void rename(const char *_logicalname,IUserDescriptor *user) override
     {
         StringBuffer prevname;
         Owned<IFileRelationshipIterator> reliter;
@@ -5561,7 +5562,7 @@ public:
     }
 
 
-    const char *queryDefaultDir()
+    virtual const char *queryDefaultDir() override
     {
         // returns the directory if all the same
         const char *ret = NULL;
@@ -5581,7 +5582,7 @@ public:
         return ret;
     }
 
-    const char *queryPartMask()
+    virtual const char *queryPartMask() override
     {
         // returns the part mask if all the same
         const char *ret = NULL;
@@ -5598,7 +5599,7 @@ public:
         return ret;
     }
 
-    void attach(const char *_logicalname,IUserDescriptor *user)
+    virtual void attach(const char *_logicalname,IUserDescriptor *user) override
     {
         assertex(!conn.get()); // already attached
         CriticalBlock block (sect);
@@ -5616,7 +5617,7 @@ public:
         loadSubFiles(NULL, 0, true);
     }
 
-    void detach(unsigned timeoutMs=INFINITE, ICodeContext *ctx=NULL)
+    virtual void detach(unsigned timeoutMs=INFINITE, ICodeContext *ctx=NULL) override
     {
         assertex(conn.get()); // must be attached
         CriticalBlock block(sect);
@@ -5645,7 +5646,7 @@ public:
         logicalName.clear();
     }
 
-    bool existsPhysicalPartFiles(unsigned short port)
+    virtual bool existsPhysicalPartFiles(unsigned short port) override
     {
         CriticalBlock block (sect);
         ForEachItemIn(i,subfiles) {
@@ -5656,7 +5657,7 @@ public:
         return true;
     }
 
-    bool renamePhysicalPartFiles(const char *newlfn,const char *cluster,IMultiException *mexcept,const char *newbasedir)
+    virtual bool renamePhysicalPartFiles(const char *newlfn,const char *cluster,IMultiException *mexcept,const char *newbasedir) override
     {
         throw MakeStringException(-1,"renamePhysicalPartFiles not supported for SuperFiles");
         return false;
@@ -5667,7 +5668,7 @@ public:
         UNIMPLEMENTED; // not yet needed
     }
 
-    virtual unsigned numCopies(unsigned partno)
+    virtual unsigned numCopies(unsigned partno) override
     {
         unsigned ret = (unsigned)-1;
         CriticalBlock block (sect);
@@ -5680,7 +5681,7 @@ public:
         return (ret==(unsigned)-1)?1:ret;
     }
 
-    __int64 getFileSize(bool allowphysical,bool forcephysical)
+    virtual __int64 getFileSize(bool allowphysical,bool forcephysical) override
     {
         __int64 ret = (__int64)(forcephysical?-1:queryAttributes().getPropInt64("@size",-1));
         if (ret==-1)
@@ -5697,7 +5698,7 @@ public:
         return ret;
     }
 
-    __int64 getDiskSize(bool allowphysical,bool forcephysical)
+    virtual __int64 getDiskSize(bool allowphysical,bool forcephysical) override
     {
         if (!isCompressed(NULL))
             return getFileSize(allowphysical, forcephysical);
@@ -5734,7 +5735,7 @@ public:
         return ret;
     }
 
-    bool getFileCheckSum(unsigned &checkSum)
+    virtual bool getFileCheckSum(unsigned &checkSum) override
     {
         if (queryAttributes().hasProp("@checkSum"))
             checkSum = (unsigned)queryAttributes().getPropInt64("@checkSum");
@@ -5751,12 +5752,12 @@ public:
         return true;
     }
 
-    IDistributedSuperFile *querySuperFile()
+    virtual IDistributedSuperFile *querySuperFile() override
     {
         return this;
     }
 
-    virtual IDistributedFile &querySubFile(unsigned idx,bool sub)
+    virtual IDistributedFile &querySubFile(unsigned idx,bool sub) override
     {
         CriticalBlock block (sect);
         if (sub) {
@@ -5777,7 +5778,7 @@ public:
         return subfiles.item(idx);
     }
 
-    virtual IDistributedFile *querySubFileNamed(const char *name, bool sub)
+    virtual IDistributedFile *querySubFileNamed(const char *name, bool sub) override
     {
         CriticalBlock block (sect);
         unsigned idx=findSubFileOrd(name);
@@ -5800,13 +5801,13 @@ public:
         return NULL;
     }
 
-    virtual IDistributedFile *getSubFile(unsigned idx,bool sub)
+    virtual IDistributedFile *getSubFile(unsigned idx,bool sub) override
     {
         CriticalBlock block (sect);
         return LINK(&querySubFile(idx,sub));
     }
 
-    virtual unsigned numSubFiles(bool sub)
+    virtual unsigned numSubFiles(bool sub) override
     {
         CriticalBlock block (sect);
         unsigned ret = 0;
@@ -5825,7 +5826,7 @@ public:
         return ret;
     }
 
-    virtual bool getFormatCrc(unsigned &crc)
+    virtual bool getFormatCrc(unsigned &crc) override
     {
         if (queryAttributes().hasProp("@formatCrc")) {
             crc = (unsigned)queryAttributes().getPropInt("@formatCrc");
@@ -5844,15 +5845,15 @@ public:
         return found;
     }
 
-    virtual bool getRecordLayout(MemoryBuffer &layout)
+    virtual bool getRecordLayout(MemoryBuffer &layout, const char *attrname) override
     {
         layout.clear();
-        if (queryAttributes().getPropBin("_record_layout",layout))
+        if (queryAttributes().getPropBin(attrname, layout))
             return true;
         bool found = false;
         ForEachItemIn(i,subfiles) {
             MemoryBuffer b;
-            if (subfiles.item(i).getRecordLayout(found?b:layout)) {
+            if (subfiles.item(i).getRecordLayout(found?b:layout, attrname)) {
                 if (found) {
                     if ((b.length()!=layout.length())||(memcmp(b.bufferBase(),layout.bufferBase(),b.length())!=0))
                         return false;
@@ -5864,7 +5865,7 @@ public:
         return found;
     }
 
-    virtual bool getRecordSize(size32_t &rsz)
+    virtual bool getRecordSize(size32_t &rsz) override
     {
         if (queryAttributes().hasProp("@recordSize")) {
             rsz = (size32_t)queryAttributes().getPropInt("@recordSize");
@@ -5883,12 +5884,12 @@ public:
         return found;
     }
 
-    virtual bool isInterleaved()
+    virtual bool isInterleaved() override
     {
         return interleaved!=0;
     }
 
-    virtual IDistributedFile *querySubPart(unsigned partidx,unsigned &subfileidx)
+    virtual IDistributedFile *querySubPart(unsigned partidx,unsigned &subfileidx) override
     {
         CriticalBlock block (sect);
         subfileidx = 0;
@@ -5910,7 +5911,7 @@ public:
         return NULL;
     }
 
-    virtual unsigned getPositionPart(offset_t pos, offset_t &base)
+    virtual unsigned getPositionPart(offset_t pos, offset_t &base) override
     {   // not very quick!
         CriticalBlock block (sect);
         unsigned n = numParts();
@@ -5928,7 +5929,7 @@ public:
         return NotFound;
     }
 
-    IDistributedFileIterator *getSubFileIterator(bool supersub=false)
+    virtual IDistributedFileIterator *getSubFileIterator(bool supersub=false) override
     {
         CriticalBlock block (sect);
         return new cSubFileIterator(subfiles,supersub);
@@ -5948,10 +5949,11 @@ public:
         root->removeProp("Attr/@size");
         root->removeProp("Attr/@compressedSize");
         root->removeProp("Attr/@checkSum");
-        root->removeProp("Attr/@recordCount");  // recordCount not currently supported by superfiles
-        root->removeProp("Attr/@formatCrc");    // formatCrc set if all consistant
-        root->removeProp("Attr/@recordSize");   // record size set if all consistant
-        root->removeProp("Attr/_record_layout");
+        root->removeProp("Attr/@recordCount");   // recordCount not currently supported by superfiles
+        root->removeProp("Attr/@formatCrc");     // formatCrc set if all consistant
+        root->removeProp("Attr/@recordSize");    // record size set if all consistant
+        root->removeProp("Attr/_record_layout"); // legacy info - set if all consistent
+        root->removeProp("Attr/_rtlType");       // new info - set if all consistent
         __int64 fs = getFileSize(false,false);
         if (fs!=-1)
             root->setPropInt64("Attr/@size",fs);
@@ -5974,9 +5976,10 @@ public:
         if (getRecordSize(rsz))
             root->setPropInt("Attr/@recordSize", rsz);
         MemoryBuffer mb;
-        if (getRecordLayout(mb))
+        if (getRecordLayout(mb, "_record_layout"))
             root->setPropBin("Attr/_record_layout", mb.length(), mb.bufferBase());
-
+        if (getRecordLayout(mb, "_rtlType"))
+            root->setPropBin("Attr/_rtlType", mb.length(), mb.bufferBase());
     }
 
     void updateParentFileAttrs(IDistributedFileTransaction *transaction)
@@ -6008,7 +6011,7 @@ public:
             throw MakeStringException(-1,"addSubFile: File %s is already a subfile of %s", sub->queryLogicalName(),queryLogicalName());
     }
 
-    void validate()
+    virtual void validate() override
     {
         unsigned numSubfiles = root->getPropInt("@numsubfiles",0);
         if (numSubfiles)
@@ -6153,12 +6156,12 @@ private:
     }
 
 public:
-    void addSubFile(const char * subfile,
-                    bool before=false,              // if true before other
-                    const char *other=NULL,     // in NULL add at end (before=false) or start(before=true)
-                    bool addcontents=false,
-                    IDistributedFileTransaction *transaction=NULL
-                   )
+    virtual void addSubFile(const char * subfile,
+                            bool before=false,              // if true before other
+                            const char *other=NULL,     // in NULL add at end (before=false) or start(before=true)
+                            bool addcontents=false,
+                            IDistributedFileTransaction *transaction=NULL
+                   ) override
     {
         CriticalBlock block (sect);
         if (!subfile||!*subfile)
@@ -6203,9 +6206,9 @@ public:
     }
 
     virtual bool removeSubFile(const char *subfile,         // if NULL removes all
-                                bool remsub,                // if true removes subfiles from DFS
-                                bool remcontents,           // if true, recurse super-files
-                                IDistributedFileTransaction *transaction)
+                               bool remsub,                // if true removes subfiles from DFS
+                               bool remcontents,           // if true, recurse super-files
+                               IDistributedFileTransaction *transaction) override
     {
         CriticalBlock block (sect);
         if (subfile&&!*subfile)
@@ -6264,7 +6267,7 @@ public:
     }
 
     virtual bool removeOwnedSubFiles(bool remsub, // if true removes subfiles from DFS
-                                     IDistributedFileTransaction *transaction)
+                                     IDistributedFileTransaction *transaction) override
     {
         CriticalBlock block (sect);
         checkModify("removeOwnedSubFiles");
@@ -6292,7 +6295,7 @@ public:
     }
 
     virtual bool swapSuperFile( IDistributedSuperFile *_file,
-                                IDistributedFileTransaction *transaction)
+                                IDistributedFileTransaction *transaction) override
     {
         CriticalBlock block (sect);
         if (!_file)
@@ -6319,7 +6322,7 @@ public:
         return true;
     }
 
-    void savePartsAttr(bool force)
+    virtual void savePartsAttr(bool force) override
     {
     }
 
@@ -6342,35 +6345,35 @@ public:
         }
     }
 
-    unsigned getClusterNames(StringArray &clusters)
+    virtual unsigned getClusterNames(StringArray &clusters) override
     {
         CriticalBlock block (sect);
         fillClustersCache();
         return clusterscache.getNames(clusters);
     }
 
-    unsigned numClusters()
+    virtual unsigned numClusters() override
     {
         CriticalBlock block (sect);
         fillClustersCache();
         return clusterscache.ordinality();
     }
 
-    unsigned findCluster(const char *clustername)
+    virtual unsigned findCluster(const char *clustername) override
     {
         CriticalBlock block (sect);
         fillClustersCache();
         return clusterscache.find(clustername);
     }
 
-    ClusterPartDiskMapSpec &queryPartDiskMapping(unsigned clusternum)
+    virtual ClusterPartDiskMapSpec &queryPartDiskMapping(unsigned clusternum) override
     {
         CriticalBlock block (sect);
         fillClustersCache();
         return clusterscache.queryPartDiskMapping(clusternum);
     }
 
-    void updatePartDiskMapping(const char *clustername,const ClusterPartDiskMapSpec &spec)
+    virtual void updatePartDiskMapping(const char *clustername,const ClusterPartDiskMapSpec &spec) override
     {
         if (!clustername||!*clustername)
             return;
@@ -6382,21 +6385,21 @@ public:
         }
     }
 
-    IGroup *queryClusterGroup(unsigned clusternum)
+    virtual IGroup *queryClusterGroup(unsigned clusternum) override
     {
         CriticalBlock block (sect);
         fillClustersCache();
         return clusterscache.queryGroup(clusternum);
     }
 
-    StringBuffer &getClusterGroupName(unsigned clusternum, StringBuffer &name)
+    virtual StringBuffer &getClusterGroupName(unsigned clusternum, StringBuffer &name) override
     {
         CriticalBlock block (sect);
         fillClustersCache();
         return clusterscache.item(clusternum).getGroupName(name, &queryNamedGroupStore());
     }
 
-    void addCluster(const char *clustername,const ClusterPartDiskMapSpec &mspec)
+    virtual void addCluster(const char *clustername,const ClusterPartDiskMapSpec &mspec) override
     {
         if (!clustername||!*clustername)
             return;
@@ -6405,7 +6408,7 @@ public:
         subfiles.item(0).addCluster(clustername,mspec);
     }
 
-    virtual bool removeCluster(const char *clustername)
+    virtual bool removeCluster(const char *clustername) override
     {
         bool clusterRemoved=false;
         CriticalBlock block (sect);
@@ -6417,7 +6420,7 @@ public:
         return clusterRemoved;
     }
 
-    void setPreferredClusters(const char *clusters)
+    virtual void setPreferredClusters(const char *clusters) override
     {
         CriticalBlock block (sect);
         clusterscache.clear();
@@ -6427,7 +6430,7 @@ public:
         }
     }
 
-    virtual bool checkClusterCompatible(IFileDescriptor &fdesc, StringBuffer &err)
+    virtual bool checkClusterCompatible(IFileDescriptor &fdesc, StringBuffer &err) override
     {
         CriticalBlock block (sect);
         if (subfiles.ordinality()!=1) {
@@ -6443,7 +6446,7 @@ public:
     }
 
 
-    void setSingleClusterOnly()
+    virtual void setSingleClusterOnly() override
     {
         CriticalBlock block (sect);
         ForEachItemIn(i,subfiles) {
@@ -6453,7 +6456,7 @@ public:
     }
 
 
-    void enqueueReplicate()
+    virtual void enqueueReplicate() override
     {
         CriticalBlock block (sect);
         ForEachItemIn(i,subfiles) {
@@ -6462,7 +6465,7 @@ public:
         }
     }
 
-    bool getAccessedTime(CDateTime &dt)
+    virtual bool getAccessedTime(CDateTime &dt) override
     {
         bool set=false;
         CriticalBlock block (sect);
@@ -6481,7 +6484,7 @@ public:
         return set;
     }
 
-    void setAccessedTime(const CDateTime &dt)
+    virtual void setAccessedTime(const CDateTime &dt) override
     {
         {
             CriticalBlock block (sect);

dali/base/dadfs.hpp

@@ -380,7 +380,7 @@ interface IDistributedFile: extends IInterface
 
     virtual bool getFormatCrc(unsigned &crc) =0;   // CRC for record format
     virtual bool getRecordSize(size32_t &rsz) =0;
-    virtual bool getRecordLayout(MemoryBuffer &layout) =0;
+    virtual bool getRecordLayout(MemoryBuffer &layout, const char *attrname) =0;
 
 
     virtual void enqueueReplicate()=0;

dali/base/dafdesc.cpp

@@ -3057,7 +3057,9 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
     if (tree->getProp("@formatCrc",fps.clear())&&fps.length())
         fprop.setProp("@formatCrc",fps.str());
     MemoryBuffer mb;
-    if (tree->getPropBin("_record_layout", mb))
+    if (tree->getPropBin("_rtlType", mb))
+        fprop.setPropBin("_rtlType", mb.length(), mb.toByteArray());
+    if (tree->getPropBin("_record_layout", mb.clear()))  // Legacy info
         fprop.setPropBin("_record_layout", mb.length(), mb.toByteArray());
     if (iskey) {
         fprop.setProp("@kind","key");

dali/base/dasds.cpp

@@ -2505,7 +2505,7 @@ public:
             serverId = 0;
         }
     }
-    virtual bool removeTree(IPropertyTree *_child)
+    virtual bool removeTree(IPropertyTree *_child) override
     {
         if (!_child)
             return false;
@@ -2518,31 +2518,31 @@ public:
         return true;
     }
 
-    virtual bool isOrphaned() const { return IptFlagTst(flags, ipt_ext5); }
+    virtual bool isOrphaned() const override { return IptFlagTst(flags, ipt_ext5); }
 
-    virtual void setServerId(__int64 _serverId)
+    virtual void setServerId(__int64 _serverId) override
     {
         if (serverId && serverId != _serverId)
             WARNLOG("Unexpected - client server id mismatch in %s, id=%" I64F "x", queryName(), _serverId);
         CRemoteTreeBase::setServerId(_serverId);
     }
 
-    virtual CSubscriberContainerList *getSubscribers(const char *xpath, CPTStack &stack)
+    virtual CSubscriberContainerList *getSubscribers(const char *xpath, CPTStack &stack) override
     {
         return SDSManager->getSubscribers(xpath, stack);
     }
 
-    IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false)
+    IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false) override
     {
         return new CServerRemoteTree(name, value, children);
     }
 
-    IPropertyTree *create(MemoryBuffer &mb)
+    IPropertyTree *create(MemoryBuffer &mb) override
     {
         return new CServerRemoteTree(mb);
     }
 
-    virtual void createChildMap() { children = new COrphanHandler(); }
+    virtual void createChildMap() override { children = new COrphanHandler(); }
 
     inline bool testExternalCandidate()
     {
@@ -2612,7 +2612,7 @@ public:
         mb.append(STIInfo);
     }
 
-    virtual void deserializeSelfRT(MemoryBuffer &src)
+    virtual void deserializeSelfRT(MemoryBuffer &src) override
     {
         CRemoteTreeBase::deserializeSelfRT(src);
         assertex(!isnocase());
@@ -2620,13 +2620,13 @@ public:
         src.read(STIInfo);
     }
 
-    virtual void removingElement(IPropertyTree *tree, unsigned pos)
+    virtual void removingElement(IPropertyTree *tree, unsigned pos) override
     {
         COrphanHandler::setOrphans(*(CServerRemoteTree *)tree, true);       
         CRemoteTreeBase::removingElement(tree, pos);
     }
 
-    virtual bool isCompressed(const char *xpath=NULL) const
+    virtual bool isCompressed(const char *xpath=NULL) const override
     {
         if (isAttribute(xpath)) return false;
         if (CRemoteTreeBase::isCompressed(xpath)) return true;
@@ -2635,7 +2635,7 @@ public:
         return child->hasProp(EXT_ATTR);
     }
 
-    bool getProp(const char *xpath, StringBuffer &ret) const
+    bool getProp(const char *xpath, StringBuffer &ret) const override
     {
         if (xpath)
             return CRemoteTreeBase::getProp(xpath, ret);

dali/base/dautils.hpp

@@ -134,7 +134,7 @@ public:
     // Multi routines
     unsigned multiOrdinality() const;
     const CDfsLogicalFileName &multiItem(unsigned idx) const;
-    const void resolveWild();  // only for multi
+    void resolveWild();  // only for multi
     IPropertyTree *createSuperTree() const;
     void allowOsPath(bool allow=true) { allowospath = allow; } // allow local OS path to be specified
     void setAllowWild(bool b=true) { allowWild = b; } // allow wildcards

ecl/eclcc/eclcc.cpp

@@ -2127,6 +2127,8 @@ IHqlExpression *EclCC::lookupDFSlayout(const char *filename, IErrorReceiver &err
                     else
                     {
                         diskRecord.set(diskRecord->queryBody());  // Remove location info - it's meaningless
+                        // MORE - if we already have the payload size info from the parsing of the ECL (we should, for all files built since 2017)
+                        // then don't parse _record_layout (which we expect to remove, eventually).
                         if (dfsFile->queryAttributes().hasProp("_record_layout"))
                         {
                             MemoryBuffer mb;

ecl/hql/hqlexpr.cpp

@@ -14062,16 +14062,16 @@ void exportMap(IPropertyTree *dataNode, IHqlExpression *destTable, IHqlExpressio
 
 void exportJsonType(StringBuffer &ret, IHqlExpression *table)
 {
-    Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
+    Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
     const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
     dumpTypeInfo(ret, typeInfo);
 }
 
 void exportBinaryType(MemoryBuffer &ret, IHqlExpression *table)
 {
-    Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
+    Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
     const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
-    dumpTypeInfo(ret, typeInfo);
+    dumpTypeInfo(ret, typeInfo, false);
 }
 
 void exportData(IPropertyTree *data, IHqlExpression *table, bool flatten)
@@ -15947,7 +15947,15 @@ unsigned numPayloadFields(IHqlExpression * index)
 
 unsigned numKeyedFields(IHqlExpression * index)
 {
-    return getFlatFieldCount(index->queryRecord())-numPayloadFields(index);
+    IHqlExpression *record = index->queryRecord();
+    unsigned fields = 0;
+    ForEachChild(idx, record)
+    {
+        IHqlExpression * child = record->queryChild(idx);
+        if (!child->isAttribute())
+            fields++;
+    }
+    return fields - numPayloadFields(index);
 }
 
 unsigned firstPayloadField(IHqlExpression * index)

ecl/hql/hqlgram2.cpp

@@ -7224,7 +7224,7 @@ IHqlExpression * HqlGram::checkIndexRecord(IHqlExpression * record, const attrib
         {
             IHqlExpression * lastField = record->queryChild(numFields-1);
             ITypeInfo * fileposType = lastField->queryType();
-            if (!isIntegralType(fileposType))
+            if (!isSimpleIntegralType(fileposType))
                 indexAttrs.setown(createComma(indexAttrs.getClear(), createExprAttribute(filepositionAtom, createConstant(false))));
         }
     }

ecl/hql/hqlir.cpp

@@ -675,7 +675,10 @@ const char * getTypeIRText(type_t type)
     EXPAND_CASE(type,decimal);
     EXPAND_CASE(type,string);
     EXPAND_CASE(type,date);
+    EXPAND_CASE(type,biasedswapint);
+    EXPAND_CASE(type,swapfilepos);
     EXPAND_CASE(type,bitfield);
+    EXPAND_CASE(type,keyedint);
     EXPAND_CASE(type,char);
     EXPAND_CASE(type,enumerated);
     EXPAND_CASE(type,record);
@@ -692,6 +695,7 @@ const char * getTypeIRText(type_t type)
     EXPAND_CASE(type,void);
     EXPAND_CASE(type,alien);
     case type_swapint: return "swap";
+    EXPAND_CASE(type,filepos);
     EXPAND_CASE(type,none);
     EXPAND_CASE(type,packedint);
     EXPAND_CASE(type,qstring);
@@ -712,12 +716,6 @@ const char * getTypeIRText(type_t type)
     EXPAND_CASE(type,sortlist);
     EXPAND_CASE(type,dictionary);
     EXPAND_CASE(type,alias);
-
-    case type_unused2:
-    case type_unused3:
-    case type_unused4:
-    case type_unused5:
-        return "unused";
     }
     return "<unknown>";
 }

ecl/hql/hqlstack.cpp

@@ -249,7 +249,7 @@ int FuncCallStack::push(ITypeInfo* argType, IHqlExpression* curParam)
 int FuncCallStack::pushMeta(ITypeInfo *type)
 {
     if (!deserializer)
-        deserializer.setown(createRtlFieldTypeDeserializer());
+        deserializer.setown(createRtlFieldTypeDeserializer(nullptr));
     const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), type);
     CDynamicOutputMetaData * meta = new CDynamicOutputMetaData(* static_cast<const RtlRecordTypeInfo *>(typeInfo));
     metas.append(*meta);

ecl/hql/hqlutil.cpp

@@ -4465,6 +4465,7 @@ IDefRecordElement * RecordMetaCreator::createIfBlock(IHqlExpression * cur, IHqlE
             return NULL;
         break;
     default:
+//        EclIR::dump_ir(cond);
         return NULL;
     }
 
@@ -9813,7 +9814,7 @@ unsigned buildRtlRecordFields(IRtlFieldTypeDeserializer &deserializer, unsigned
         switch (field->getOperator())
         {
         case no_ifblock:
-            typeFlags |= (RFTMnoserialize|RFTMnoprefetch);
+            typeFlags |= (RFTMunknownsize|RFTMnoprefetch);
             break;
         case no_field:
         {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3762,7 +3762,7 @@ unsigned HqlCppTranslator::buildRtlIfBlockField(StringBuffer & instanceName, IHq
     BuildCtx declarectx(*code, declareAtom);
 
     //First generate a pseudo type entry for an ifblock.
-    unsigned fieldType = type_ifblock|RFTMcontainsifblock|RFTMnoserialize|RFTMunknownsize|RFTMnoprefetch;
+    unsigned fieldType = type_ifblock|RFTMunknownsize|RFTMnoprefetch;
     {
         unsigned length = 0;
         StringBuffer childTypeName;

ecl/hthor/hthor.cpp

@@ -48,6 +48,7 @@
 #include "eclagent.ipp"
 #include "roxierowbuff.hpp"
 #include "ftbase.ipp"
+#include "rtldynfield.hpp"
 
 #define EMPTY_LOOP_LIMIT 1000
 
@@ -1123,10 +1124,7 @@ void CHThorIndexWriteActivity::execute()
             }
             reccount++;
         }
-        if(metadata)
-            builder->finish(metadata,&fileCrc);
-        else
-            builder->finish(&fileCrc);
+        builder->finish(metadata, &fileCrc);
         out->flush();
         out.clear();
     }
@@ -1186,11 +1184,6 @@ void CHThorIndexWriteActivity::execute()
     properties.setProp("@owner", agent.queryWorkUnit()->queryUser());
     properties.setProp("@workunit", agent.queryWorkUnit()->queryWuid());
     properties.setProp("@job", agent.queryWorkUnit()->queryJobName());
-#if 0
-    IRecordSize * irecsize = helper.queryDiskRecordSize();
-    if(irecsize && (irecsize->isFixedSize()))
-        properties.setPropInt("@recordSize", irecsize->getFixedSize());
-#endif
     char const * rececl = helper.queryRecordECL();
     if(rececl && *rececl)
         properties.setProp("ECL", rececl);
@@ -1209,6 +1202,7 @@ void CHThorIndexWriteActivity::execute()
 
     properties.setPropInt("@fileCrc", fileCrc);
     properties.setPropInt("@formatCrc", helper.getFormatCrc());
+    // Legacy record layout info
     void * layoutMetaBuff;
     size32_t layoutMetaSize;
     if(helper.getIndexLayout(layoutMetaSize, layoutMetaBuff))
@@ -1216,6 +1210,14 @@ void CHThorIndexWriteActivity::execute()
         properties.setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
         rtlFree(layoutMetaBuff);
     }
+    // New record layout info
+    if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
+    {
+        MemoryBuffer out;
+        dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+        properties.setPropBin("_rtlType", out.length(), out.toByteArray());
+    }
+
     StringBuffer lfn;
     Owned<IDistributedFile> dfile = NULL;
     if (!agent.queryResolveFilesLocally())
@@ -1279,12 +1281,11 @@ void CHThorIndexWriteActivity::buildLayoutMetadata(Owned<IPropertyTree> & metada
     if(!metadata) metadata.setown(createPTree("metadata"));
     metadata->setProp("_record_ECL", helper.queryRecordECL());
 
-    void * layoutMetaBuff;
-    size32_t layoutMetaSize;
-    if(helper.getIndexLayout(layoutMetaSize, layoutMetaBuff))
+    if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
     {
-        metadata->setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
-        rtlFree(layoutMetaBuff);
+        MemoryBuffer out;
+        dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+        metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
     }
 }
 

ecl/hthor/hthorkey.cpp

@@ -27,7 +27,6 @@
 #include "roxiedebug.hpp"
 
 #define MAX_FETCH_LOOKAHEAD 1000
-#define IGNORE_FORMAT_CRC_MISMATCH_WHEN_NO_METADATA
 
 using roxiemem::IRowManager;
 using roxiemem::OwnedRoxieRow;
@@ -95,15 +94,7 @@ IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activ
     IPropertyTree const & props = df->queryAttributes();
     MemoryBuffer diskMetaBuff;
     if(!props.getPropBin("_record_layout", diskMetaBuff))
-#ifdef IGNORE_FORMAT_CRC_MISMATCH_WHEN_NO_METADATA
-    {
-        WARNLOG("On reading index %s, formatCRC mismatch ignored because file had no record layout metadata and so assumed old", df->queryLogicalName());
-        return NULL;
-    }
-#else
         throw MakeStringException(0, "Unable to recover from record layout mismatch for index %s: no record layout metadata in file", df->queryLogicalName());
-#endif
-
     try
     {
         if(cache)

plugins/javaembed/javaembed.cpp

@@ -30,7 +30,7 @@
 #include "build-config.h"
 #include "roxiemem.hpp"
 #include "nbcd.hpp"
-#include "thorxmlwrite.hpp"
+#include "rtlformat.hpp"
 #include "esdl_def.hpp"
 
 #ifndef _WIN32

roxie/ccd/ccdserver.cpp

@@ -74,6 +74,7 @@
 #include "thorplugin.hpp"
 #include "keybuild.hpp"
 #include "thorstrand.hpp"
+#include "rtldynfield.hpp"
 
 #define MAX_HTTP_HEADERSIZE 8000
 #define MIN_PAYLOAD_SIZE 800
@@ -12013,12 +12014,11 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
             metadata.setown(createPTree("metadata", ipt_fast));
         metadata->setProp("_record_ECL", helper.queryRecordECL());
 
-        void * layoutMetaBuff;
-        size32_t layoutMetaSize;
-        if(helper.getIndexLayout(layoutMetaSize, layoutMetaBuff))
+        if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
         {
-            metadata->setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
-            rtlFree(layoutMetaBuff);
+            MemoryBuffer out;
+            dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+            metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }
 
@@ -12119,10 +12119,7 @@ public:
                 }
                 reccount++;
             }
-            if(metadata)
-                builder->finish(metadata,&fileCrc);
-            else
-                builder->finish(&fileCrc);
+            builder->finish(metadata, &fileCrc);
         }
     }
 
@@ -12217,6 +12214,7 @@ public:
 
         properties.setPropInt("@fileCrc", fileCrc);
         properties.setPropInt("@formatCrc", helper.getFormatCrc());
+        // Legacy record layout info
         void * layoutMetaBuff;
         size32_t layoutMetaSize;
         if(helper.getIndexLayout(layoutMetaSize, layoutMetaBuff))
@@ -12224,6 +12222,13 @@ public:
             properties.setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
             rtlFree(layoutMetaBuff);
         }
+        // New record layout info
+        if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
+        {
+            MemoryBuffer out;
+            dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+            properties.setPropBin("_rtlType", out.length(), out.toByteArray());
+        }
     }
 
     IUserDescriptor *queryUserDescriptor() const
@@ -23724,12 +23729,15 @@ public:
         sorted = (flags & TIRunordered) == 0;
         isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
         rtlDataAttr indexLayoutMeta;
-        size32_t indexLayoutSize;
-        if(!indexHelper->getIndexLayout(indexLayoutSize, indexLayoutMeta.refdata()))
+        size32_t indexLayoutSize = 0;
+        if(indexHelper->getIndexLayout(indexLayoutSize, indexLayoutMeta.refdata()))
+        {
+            MemoryBuffer m;
+            m.setBuffer(indexLayoutSize, indexLayoutMeta.getdata());
+            activityMeta.setown(deserializeRecordMeta(m, true));
+        }
+        else
             assertex(indexLayoutSize==0);
-        MemoryBuffer m;
-        m.setBuffer(indexLayoutSize, indexLayoutMeta.getdata());
-        activityMeta.setown(deserializeRecordMeta(m, true));
         enableFieldTranslation = queryFactory.queryOptions().enableFieldTranslation;
         translatorArray.setown(new TranslatorArray);
         variableFileName = allFilesDynamic || _queryFactory.isDynamic() || ((flags & (TIRvarfilename|TIRdynamicfilename)) != 0);

rtl/eclrtl/eclhelper_dyn.cpp

@@ -40,9 +40,9 @@
 class CDeserializedOutputMetaData : public COutputMetaData
 {
 public:
-    CDeserializedOutputMetaData(MemoryBuffer &binInfo);
-    CDeserializedOutputMetaData(IPropertyTree &jsonInfo);
-    CDeserializedOutputMetaData(const char *json);
+    CDeserializedOutputMetaData(MemoryBuffer &binInfo, IThorIndexCallback *callback);
+    CDeserializedOutputMetaData(IPropertyTree &jsonInfo, IThorIndexCallback *callback);
+    CDeserializedOutputMetaData(const char *json, IThorIndexCallback *callback);
 
     virtual const RtlTypeInfo * queryTypeInfo() const override { return typeInfo; }
 protected:
@@ -50,55 +50,159 @@ protected:
     const RtlTypeInfo *typeInfo = nullptr;
 };
 
-
-CDeserializedOutputMetaData::CDeserializedOutputMetaData(MemoryBuffer &binInfo)
+CDeserializedOutputMetaData::CDeserializedOutputMetaData(MemoryBuffer &binInfo, IThorIndexCallback *callback)
 {
-    deserializer.setown(createRtlFieldTypeDeserializer());
+    deserializer.setown(createRtlFieldTypeDeserializer(callback));
     typeInfo = deserializer->deserialize(binInfo);
 }
 
-CDeserializedOutputMetaData::CDeserializedOutputMetaData(IPropertyTree &jsonInfo)
+CDeserializedOutputMetaData::CDeserializedOutputMetaData(IPropertyTree &jsonInfo, IThorIndexCallback *callback)
 {
-    deserializer.setown(createRtlFieldTypeDeserializer());
+    deserializer.setown(createRtlFieldTypeDeserializer(callback));
     typeInfo = deserializer->deserialize(jsonInfo);
 }
 
-CDeserializedOutputMetaData::CDeserializedOutputMetaData(const char *json)
+CDeserializedOutputMetaData::CDeserializedOutputMetaData(const char *json, IThorIndexCallback *callback)
 {
-    deserializer.setown(createRtlFieldTypeDeserializer());
+    deserializer.setown(createRtlFieldTypeDeserializer(callback));
     typeInfo = deserializer->deserialize(json);
 }
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &binInfo)
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &binInfo, IThorIndexCallback *callback)
 {
-    return new CDeserializedOutputMetaData(binInfo);
+    return new CDeserializedOutputMetaData(binInfo, callback);
 }
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonInfo)
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonInfo, IThorIndexCallback *callback)
 {
-    return new CDeserializedOutputMetaData(jsonInfo);
+    return new CDeserializedOutputMetaData(jsonInfo, callback);
 }
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json)
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, IThorIndexCallback *callback)
 {
-    return new CDeserializedOutputMetaData(json);
+    return new CDeserializedOutputMetaData(json, callback);
 }
 //---------------------------------------------------------------------------------------------------------------------
 
+static int compareOffsets(const unsigned *a, const unsigned *b)
+{
+    if (*a < *b)
+        return -1;
+    else if (*a==*b)
+        return 0;
+    else
+        return 1;
+}
+
+class FilterSet
+{
+public:
+    FilterSet(const RtlRecord &_inrec) : inrec(_inrec)
+    {
+
+    }
+    void addFilter(const char *filter)
+    {
+        // Format of a filter is:
+        // field[..n]: valuestring
+        // value string format specifies ranges using a comma-separated list of ranges.
+        // Each range is specified as paren lower, upper paren, where the paren is either ( or [ depending
+        // on whether the specified bound is inclusive or exclusive.
+        // If only one bound is specified then it is used for both upper and lower bound (only meaningful with [] )
+        //
+        // ( A means values > A - exclusive
+        // [ means values >= A - inclusive
+        // A ) means values < A - exclusive
+        // A ] means values <= A - inclusive
+        // For example:
+        // [A] matches just A
+        // (,A),(A,) matches all but A
+        // (A] of [A) are both empty ranges
+        // [A,B) means A*
+        // Values use the ECL syntax for constants. String constants are always utf8. Binary use d'xx' format (hexpairs)
+        // Note that binary serialization format is different
+
+        assertex(filter);
+        const char *epos = strpbrk(filter,"=~");
+        if (!epos)
+            throw MakeStringException(0, "Invalid filter string: expected = or ~ after fieldname");
+        StringBuffer fieldName(epos-filter, filter);
+        unsigned fieldNum = inrec.getFieldNum(fieldName);
+        if (fieldNum == (unsigned) -1)
+            throw MakeStringException(0, "Invalid filter string: field '%s' not recognized", fieldName.str());
+        unsigned numOffsets = inrec.getNumVarFields() + 1;
+        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow offsetCalculator(inrec, nullptr, numOffsets, variableOffsets);
+        unsigned fieldOffset = offsetCalculator.getOffset(fieldNum);
+        unsigned fieldSize = offsetCalculator.getSize(fieldNum);
+        const RtlTypeInfo *fieldType = inrec.queryType(fieldNum);
+        filter = epos+1;
+        if (*filter=='~')
+        {
+            UNIMPLEMENTED;  // use a regex?
+        }
+        else
+        {
+            MemoryBuffer lobuffer;
+            MemoryBuffer hibuffer;
+            Owned<IStringSet> filterSet = createStringSet(fieldSize);
+            deserializeSet(*filterSet, inrec.getMinRecordSize(), fieldType, filter);
+            aindex_t found = filterOffsets.bSearch(fieldOffset, compareOffsets);
+            if (found==NotFound)
+            {
+                bool isNew;
+                unsigned insertPos = filterOffsets.bAdd(fieldOffset, compareOffsets, isNew);
+                dbgassertex(isNew);
+                filters.add(*filterSet.getClear(), insertPos);
+            }
+            else
+            {
+                filterSet.setown(filters.item(found).unionSet(filterSet));// Debatable - would intersect be more appropriate?
+                filters.replace(*filterSet.getClear(), found);
+            }
+        }
+    }
+    void createSegmentMonitors(IIndexReadContext *irc)
+    {
+        ForEachItemIn(idx, filters)
+        {
+            IStringSet &filter = filters.item(idx);
+            irc->append(createKeySegmentMonitor(false, LINK(&filter), filterOffsets.item(idx), filter.getSize()));
+        }
+    }
+    void createSegmentMonitorsWithWild(IIndexReadContext *irc, unsigned keySize)
+    {
+        unsigned lastOffset = 0;
+        ForEachItemIn(idx, filters)
+        {
+            IStringSet &filter = filters.item(idx);
+            unsigned offset =  filterOffsets.item(idx);
+            unsigned size = filter.getSize();
+            if (offset > lastOffset)
+                irc->append(createWildKeySegmentMonitor(lastOffset, offset-lastOffset));
+            irc->append(createKeySegmentMonitor(false, LINK(&filter), offset, size));
+            lastOffset = offset+size;
+        }
+        if (keySize > lastOffset)
+            irc->append(createWildKeySegmentMonitor(lastOffset, keySize-lastOffset));
+    }
+protected:
+    IArrayOf<IStringSet> filters;
+    UnsignedArray filterOffsets;
+    const RtlRecord &inrec;
+};
+
 class ECLRTL_API CDynamicDiskReadArg : public CThorDiskReadArg
 {
 public:
     CDynamicDiskReadArg(const char *_fileName, IOutputMetaData *_in, IOutputMetaData *_out, unsigned __int64 _chooseN, unsigned __int64 _skipN, unsigned __int64 _rowLimit)
-        : fileName(_fileName), in(_in), out(_out), chooseN(_chooseN), skipN(_skipN), rowLimit(_rowLimit)
+        : fileName(_fileName), in(_in), out(_out), chooseN(_chooseN), skipN(_skipN), rowLimit(_rowLimit), filters(in->queryRecordAccessor(true))
     {
-        inrec = &in->queryRecordAccessor(true);
-        numOffsets = inrec->getNumVarFields() + 1;
-        translator.setown(createRecordTranslator(queryOutputMeta()->queryRecordAccessor(true), *inrec));
+        translator.setown(createRecordTranslator(out->queryRecordAccessor(true), in->queryRecordAccessor(true)));
     }
     virtual bool needTransform() override
     {
         return true;
-        //return translator->needsTranslate(); might be more appropriate?
     }
     virtual unsigned getFlags() override
     {
@@ -106,11 +210,7 @@ public:
     }
     virtual void createSegmentMonitors(IIndexReadContext *irc) override
     {
-        ForEachItemIn(idx, filters)
-        {
-            IStringSet &filter = filters.item(idx);
-            irc->append(createKeySegmentMonitor(false, LINK(&filter), filterOffsets.item(idx), filter.getSize()));
-        }
+        filters.createSegmentMonitors(irc);
     }
 
     virtual IOutputMetaData * queryOutputMeta() override
@@ -135,67 +235,83 @@ public:
     }
     virtual unsigned __int64 getChooseNLimit() { return chooseN; }
     virtual unsigned __int64 getRowLimit() { return rowLimit; }
-
     void addFilter(const char *filter)
     {
-        // Format of a filter is:
-        // field[..n]: valuestring
-        // value string format specifies ranges using a comma-separated list of ranges.
-        // Each range is specified as paren lower, upper paren, where the paren is either ( or [ depending
-        // on whether the specified bound is inclusive or exclusive.
-        // If only one bound is specified then it is used for both upper and lower bound (only meaningful with [] )
-        //
-        // ( A means values > A - exclusive
-        // [ means values >= A - inclusive
-        // A ) means values < A - exclusive
-        // A ] means values <= A - inclusive
-        // For example:
-        // [A] matches just A
-        // (,A),(A,) matches all but A
-        // (A] of [A) are both empty ranges
-        // [A,B) means A*
-        // Values use the ECL syntax for constants. String constants are always utf8. Binary use d'xx' format (hexpairs)
-        // Note that binary serialization format is different
+        filters.addFilter(filter);
+        flags |= TDRkeyed;
+    }
+private:
+    StringAttr fileName;
+    unsigned flags = 0;
+    Owned<IOutputMetaData> in;
+    Owned<IOutputMetaData> out;
+    Owned<const IDynamicTransform> translator;
+    FilterSet filters;
+    unsigned __int64 chooseN = I64C(0x7fffffffffffffff); // constant(s) should be commoned up somewhere
+    unsigned __int64 skipN = 0;
+    unsigned __int64 rowLimit = (unsigned __int64) -1;
+};
 
-        assertex(filter);
-        const char *epos = strchr(filter,'=');
-        assertex(epos);
-        StringBuffer fieldName(epos-filter, filter);
-        unsigned fieldNum = inrec->getFieldNum(fieldName);
-        assertex(fieldNum != (unsigned) -1);
-        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
-        RtlRow offsetCalculator(*inrec, nullptr, numOffsets, variableOffsets);
-        unsigned fieldOffset = offsetCalculator.getOffset(fieldNum);
-        unsigned fieldSize = offsetCalculator.getSize(fieldNum);
-        const RtlTypeInfo *fieldType = inrec->queryType(fieldNum);
-        filter = epos+1;
-        if (*filter=='~')
-        {
-            UNIMPLEMENTED;  // use a regex?
-        }
-        else
+class ECLRTL_API CDynamicIndexReadArg : public CThorIndexReadArg, implements IDynamicIndexReadArg
+{
+public:
+    CDynamicIndexReadArg(const char *_fileName, IOutputMetaData *_in, IOutputMetaData *_out, unsigned __int64 _chooseN, unsigned __int64 _skipN, unsigned __int64 _rowLimit)
+        : fileName(_fileName), in(_in), out(_out), chooseN(_chooseN), skipN(_skipN), rowLimit(_rowLimit), filters(in->queryRecordAccessor(true))
+    {
+        translator.setown(createRecordTranslator(out->queryRecordAccessor(true), in->queryRecordAccessor(true)));
+        if (!translator->canTranslate())
         {
-            MemoryBuffer lobuffer;
-            MemoryBuffer hibuffer;
-            Owned<IStringSet> filterSet = createStringSet(fieldSize);
-            deserializeSet(*filterSet, inrec->getMinRecordSize(), fieldType, filter);
-            filters.append(*filterSet.getClear());
-            filterOffsets.append(fieldOffset);
-            flags |= TDRkeyed;
+            translator->describe();
+            throw makeStringException(0, "Translation not possible");
         }
     }
+    virtual bool needTransform() override
+    {
+        return true;
+    }
+    virtual unsigned getFlags() override
+    {
+        return flags;
+    }
+    virtual void createSegmentMonitors(IIndexReadContext *irc) override
+    {
+        filters.createSegmentMonitorsWithWild(irc, 0); // Should be the total keyed size, but that's not available. And probably should not really be needed. Why should I create trailing wildsegs?
+    }
+
+    virtual IOutputMetaData * queryOutputMeta() override
+    {
+        return out;
+    }
+    virtual const char * getFileName() override final
+    {
+        return fileName;
+    }
+    virtual IOutputMetaData * queryDiskRecordSize() override final
+    {
+        return in;
+    }
+    virtual unsigned getFormatCrc() override
+    {
+        return 0;  // engines should treat 0 as 'ignore'
+    }
+    virtual size32_t transform(ARowBuilder & rowBuilder, const void * src) override
+    {
+        return translator->translate(rowBuilder, (const byte *) src);
+    }
+    virtual unsigned __int64 getChooseNLimit() { return chooseN; }
+    virtual unsigned __int64 getRowLimit() { return rowLimit; }
+    virtual void addFilter(const char *filter) override
+    {
+        filters.addFilter(filter);
+    }
+
 private:
     StringAttr fileName;
-    unsigned numOffsets = 0;
     unsigned flags = 0;
-    Owned<IRtlFieldTypeDeserializer> indeserializer;   // Owns the resulting ITypeInfo structures, so needs to be kept around
-    Owned<IRtlFieldTypeDeserializer> outdeserializer;  // Owns the resulting ITypeInfo structures, so needs to be kept around
     Owned<IOutputMetaData> in;
     Owned<IOutputMetaData> out;
-    IArrayOf<IStringSet> filters;
-    UnsignedArray filterOffsets;
-    const RtlRecord *inrec = nullptr;
     Owned<const IDynamicTransform> translator;
+    FilterSet filters;
     unsigned __int64 chooseN = I64C(0x7fffffffffffffff); // constant(s) should be commoned up somewhere
     unsigned __int64 skipN = 0;
     unsigned __int64 rowLimit = (unsigned __int64) -1;
@@ -219,7 +335,7 @@ static IOutputMetaData *loadTypeInfo(IPropertyTree &xgmml, const char *key)
     MemoryBuffer binInfo;
     xgmml.getPropBin(xpath.setf("att[@name='%s_binary']/value", key), binInfo);
     assertex(binInfo.length());
-    return new CDeserializedOutputMetaData(binInfo);
+    return new CDeserializedOutputMetaData(binInfo, nullptr);
 }
 
 extern ECLRTL_API IHThorDiskReadArg *createDiskReadArg(IPropertyTree &xgmml)
@@ -242,6 +358,12 @@ extern ECLRTL_API IHThorDiskReadArg *createDiskReadArg(const char *fileName, IOu
     return new CDynamicDiskReadArg(fileName, in, out, chooseN, skipN, rowLimit);
 }
 
+extern ECLRTL_API IHThorIndexReadArg *createIndexReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit)
+{
+    return new CDynamicIndexReadArg(fileName, in, out, chooseN, skipN, rowLimit);
+}
+
+
 extern ECLRTL_API IHThorArg *createWorkunitWriteArg(IPropertyTree &xgmml)
 {
     Owned <IOutputMetaData> in = loadTypeInfo(xgmml, "input");

rtl/eclrtl/eclhelper_dyn.hpp

@@ -22,12 +22,18 @@
 #include "eclrtl.hpp"
 #include "eclhelper.hpp"
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &mb);
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonTree);
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &mb, IThorIndexCallback *callback=nullptr);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonTree, IThorIndexCallback *callback=nullptr);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, IThorIndexCallback *callback=nullptr);
+
+interface IDynamicIndexReadArg
+{
+    virtual void addFilter(const char *filter) = 0;
+};
 
 extern ECLRTL_API IHThorDiskReadArg *createDiskReadArg(IPropertyTree &xgmml);
 extern ECLRTL_API IHThorDiskReadArg *createDiskReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit);
+extern ECLRTL_API IHThorIndexReadArg *createIndexReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit);
 extern ECLRTL_API IHThorArg *createWorkunitWriteArg(IPropertyTree &xgmml);
 extern ECLRTL_API IEclProcess* createDynamicEclProcess();
 

rtl/eclrtl/rtldynfield.cpp

@@ -30,7 +30,7 @@
 
 //---------------------------------------------------------------------------------------------------------------------
 
-const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo() const
+const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_callback) const
 {
     const RtlTypeInfo *ret = nullptr;
     switch (fieldType & RFTMkind)
@@ -38,9 +38,15 @@ const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo() const
     case type_boolean:
         ret = new RtlBoolTypeInfo(fieldType, length);
         break;
+    case type_keyedint:
+        ret = new RtlKeyedIntTypeInfo(fieldType, length, childType);
+        break;
     case type_int:
         ret = new RtlIntTypeInfo(fieldType, length);
         break;
+    case type_filepos:
+        ret = new RtlFileposTypeInfo(fieldType, length, childType, _callback);
+        break;
     case type_real:
         ret = new RtlRealTypeInfo(fieldType, length);
         break;
@@ -92,6 +98,9 @@ const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo() const
     case type_record:
         ret = new RtlRecordTypeInfo(fieldType, length, fieldsArray);
         break;
+    case type_ifblock:
+        ret = new RtlDynamicIfBlockTypeInfo(fieldType, length, fieldsArray);
+        break;
     default:
         throwUnexpected();
     }
@@ -172,7 +181,7 @@ private:
     void serializeMe(const RtlTypeInfo *type)
     {
         if (!type->canSerialize())
-            throw makeStringException(MSGAUD_user, 1000, "IFBLOCK and DICTIONARY type structures cannot be serialized");
+            throw makeStringException(MSGAUD_user, 1000, "DICTIONARY type structures cannot be serialized");
         addPropHex("fieldType", type->fieldType);
         addProp("length", type->length);
         addPropNonEmpty("locale", type->queryLocale());
@@ -295,6 +304,127 @@ private:
     bool commaPending = false;
 };
 
+class IndexBiasTranslator
+{
+public:
+    IndexBiasTranslator(const RtlTypeInfo *type)
+    {
+        translatedType = type;  // Assume no translation needed until proven otherwise
+        if (type->getType() != type_record)
+            return;
+        const RtlFieldInfo * const * fields = type->queryFields();
+        if (!fields)
+            return;
+
+        unsigned numFields;
+        needsTranslation = false;
+        for (numFields=0;;numFields++)
+        {
+            const RtlFieldInfo * child = fields[numFields];
+            if (!child)
+                break;
+            switch (child->type->getType())
+            {
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+            case type_swapint:
+                if (!child->type->isUnsigned())
+                    needsTranslation = true;
+                break;
+            case type_int:
+                needsTranslation = true;
+                break;
+#else
+            case type_int:
+                if (!child->type->isUnsigned())
+                    needsTranlsation = true;
+                break;
+            case type_swapint:
+                needsTranslation = true;
+                break;
+#endif
+            }
+        }
+        if (!needsTranslation && numFields > 1)
+        {
+            // Check if need last field translating to a type_filepos
+            switch(fields[numFields]->type->getType())
+            {
+            case type_int:
+            case type_swapint:
+            case type_packedint:
+            case type_bitfield:
+                needsTranslation = true;
+            }
+        }
+        if (needsTranslation)
+        {
+            translated = new bool[numFields];
+            RtlFieldInfo * * newFields = new RtlFieldInfo * [numFields+1];
+            newFields[numFields] = nullptr;
+            for (unsigned idx = 0; idx < numFields; idx++)
+            {
+                newFields[idx] = new RtlFieldInfo(*fields[idx]);
+                const RtlTypeInfo *newType = createBiasType(fields[idx]->type, idx > 1 && idx == numFields-1);
+                // MORE - Is it an issue if we don't common these up?
+                if (newType)
+                {
+                    newFields[idx]->type = newType;
+                    translated[idx] = true;
+                }
+                else
+                    translated[idx] = false;
+            }
+            translatedType = new RtlRecordTypeInfo(type->fieldType, type->length, newFields);
+        }
+    }
+    ~IndexBiasTranslator()
+    {
+        if (needsTranslation)
+        {
+            const RtlFieldInfo * const * fields = translatedType->queryFields();
+            for (unsigned idx = 0;;idx++)
+            {
+                const RtlFieldInfo * child = fields[idx];
+                if (!child)
+                    break;
+                if (translated[idx])
+                    child->type->doDelete();
+                delete child;
+            }
+            delete [] fields;
+            translatedType->doDelete();
+            delete [] translated;
+        }
+    }
+    const RtlTypeInfo *queryTranslatedType()
+    {
+        return translatedType;
+    }
+private:
+    static const RtlTypeInfo *createBiasType(const RtlTypeInfo *origType, bool isLastField)
+    {
+        auto type = origType->getType();
+        if (type==type_int || type==type_swapint)
+        {
+            unsigned flags = origType->fieldType & ~RFTMkind;
+            unsigned length = origType->length;
+            if (isLastField)
+                return new RtlFileposTypeInfo(type_filepos | flags, length, origType, nullptr);
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+            else if (type == type_int || origType->isSigned())
+#else
+            else if (type == type_swapint || origType->isSigned()) // MORE - this may not be right if compiler machine endianness does not match this machine
+#endif
+                return new RtlKeyedIntTypeInfo(type_keyedint | flags, length, origType);
+        }
+        return nullptr;
+    }
+
+    const RtlTypeInfo *translatedType = nullptr;
+    bool needsTranslation = false;
+    bool *translated = nullptr;
+};
+
 class CRtlFieldTypeBinSerializer
 {
 public:
@@ -305,10 +435,16 @@ public:
      * @param  type RtlTypeInfo structure to be serialized
      * @return Referenced to supplied buffer
      */
-     static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type)
+     static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type, bool applyBias)
      {
          CRtlFieldTypeBinSerializer s(out);
-         s.serializeType(type);
+         if (applyBias)
+         {
+             IndexBiasTranslator translator(type);
+             s.serializeType(translator.queryTranslatedType());
+         }
+         else
+             s.serializeType(type);
          return out;
      }
 private:
@@ -327,13 +463,12 @@ private:
             const RtlFieldInfo * const * fields = type->queryFields();
             if (fields)
             {
-                for (;;)
+                for (unsigned idx = 0;;idx++)
                 {
-                    const RtlFieldInfo * child = *fields;
+                    const RtlFieldInfo * child = fields[idx];
                     if (!child)
                         break;
                     serializeType(child->type);
-                    fields++;
                 }
             }
             // Now serialize this one
@@ -341,11 +476,11 @@ private:
             serializeMe(type);
         }
     }
-
+    
     void serializeMe(const RtlTypeInfo *type)
     {
         if (!type->canSerialize())
-            throw makeStringException(MSGAUD_user, 1000, "IFBLOCK and DICTIONARY type structures cannot be serialized");
+            throw makeStringException(MSGAUD_user, 1000, "DICTIONARY type structures cannot be serialized");
         unsigned fieldType = type->fieldType;
         const char *locale = type->queryLocale();
         if (locale && *locale)
@@ -423,8 +558,10 @@ public:
     /**
      * CRtlFieldTypeDeserializer constructor
      *
+     * @param  _callback Supplies a callback to be used for blobs/filepositions.
      */
-    CRtlFieldTypeDeserializer()
+    CRtlFieldTypeDeserializer(IThorIndexCallback *_callback)
+    : callback(_callback)
     {
     }
     /**
@@ -492,7 +629,7 @@ public:
      * <p>
      * Do not call more than once.
      *
-     * @param  _json JSON text to be deserialized, as created by CRtlFieldTypeSerializer
+     * @param  buf Binary information to be deserialized, as created by CRtlFieldTypeSerializer
      * @return Deserialized type object
      */
     virtual const RtlTypeInfo *deserialize(MemoryBuffer &buf) override
@@ -518,7 +655,7 @@ public:
         if (found)
             return *found;
         info.locale = keep(info.locale);
-        const RtlTypeInfo * ret = info.createRtlTypeInfo();
+        const RtlTypeInfo * ret = info.createRtlTypeInfo(callback);
         types.setValue(name, ret);
         return ret;
     }
@@ -542,7 +679,7 @@ private:
     KeptAtomTable atoms;     // Used to ensure proper lifetime of strings used in type structures
     MapStringTo<const RtlTypeInfo *> types;  // Ensures structures only generated once
     const RtlTypeInfo *base = nullptr;       // Holds the resulting type
-
+    IThorIndexCallback *callback = nullptr;
     void cleanupType(const RtlTypeInfo *type)
     {
         if (type)
@@ -633,7 +770,7 @@ private:
                 n++;
             }
         }
-        return info.createRtlTypeInfo();
+        return info.createRtlTypeInfo(callback);
     }
 
     const RtlTypeInfo *deserializeType(MemoryBuffer &type)
@@ -683,13 +820,33 @@ private:
             }
         }
         info.fieldType &= ~RFTMserializerFlags;
-        return info.createRtlTypeInfo();
+        return info.createRtlTypeInfo(callback);
+    }
+    void patchIndexFilePos()
+    {
+        if (callback && (base->fieldType & RFTMkind) == type_record)
+        {
+            // Yukky hack time
+            // Assumes that the fieldinfo is not shared...
+            // But that is also assumed by the code that cleans them up.
+            const RtlFieldInfo * const *fields = base->queryFields();
+            for(;;)
+            {
+                const RtlFieldInfo *field = *fields++;
+                if (!field)
+                    break;
+                if (field->type->getType() == type_filepos)  // probably blobs too?
+                {
+                    static_cast<RtlFileposTypeInfo *>(const_cast<RtlTypeInfo *>(field->type))->setCallback(callback);
+                }
+            }
+        }
     }
 };
 
-extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer()
+extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback)
 {
-    return new CRtlFieldTypeDeserializer;
+    return new CRtlFieldTypeDeserializer(callback);
 }
 
 extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t)
@@ -697,9 +854,9 @@ extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInf
     return CRtlFieldTypeSerializer::serialize(ret, t);
 }
 
-extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
+extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t, bool useBias)
 {
-    return CRtlFieldTypeBinSerializer::serialize(ret, t);
+    return CRtlFieldTypeBinSerializer::serialize(ret, t, useBias);
 }
 
 extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &metaVal)
@@ -709,12 +866,12 @@ extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,I
 
 #ifdef _DEBUG
     StringBuffer ret2;
-    CRtlFieldTypeDeserializer deserializer;
+    CRtlFieldTypeDeserializer deserializer(nullptr);
     CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
     assert(streq(ret, ret2));
     MemoryBuffer out;
-    CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
-    CRtlFieldTypeDeserializer bindeserializer;
+    CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo(), false);
+    CRtlFieldTypeDeserializer bindeserializer(nullptr);
     CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
     assert(streq(ret, ret2));
 #endif
@@ -743,7 +900,7 @@ extern ECLRTL_API int getFieldNum(const char *fieldName, IOutputMetaData &  meta
     return r.getFieldNum(fieldName);
 }
 enum FieldMatchType {
-    // On a field, exactly one of the above is set, but translator returns a bitmap indicating
+    // On a field, exactly one of the below is set, but translator returns a bitmap indicating
     // which were required (and we can restrict translation to allow some types but not others)
     match_perfect     = 0x00,    // exact type match - use memcpy
     match_link        = 0x01,    // copy a nested dataset by linking
@@ -755,6 +912,9 @@ enum FieldMatchType {
     match_none        = 0x40,    // No matching field in source - use null value
     match_recurse     = 0x80,    // Use recursive translator for child records/datasets
     match_fail        = 0x100,   // no translation possible
+
+    // This flag may be set in conjunction with the others
+    match_inifblock   = 0x200,   // matching to a field in an ifblock - may not be present
 };
 
 StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
@@ -770,6 +930,7 @@ StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
     if (flags & match_typecast) out.append("|typecast");
     if (flags & match_none) out.append("|none");
     if (flags & match_recurse) out.append("|recurse");
+    if (flags & match_inifblock) out.append("|ifblock");
     if (flags & match_fail) out.append("|fail");
     assertex(out.length() > origlen);
     return out.remove(origlen, 1);
@@ -797,6 +958,7 @@ public:
     }
     virtual size32_t translate(ARowBuilder &builder, const byte *sourceRec) const override
     {
+        dbgassertex(canTranslate());
         return doTranslate(builder, 0, sourceRec);
     }
     virtual bool canTranslate() const override
@@ -819,7 +981,7 @@ private:
             else
             {
                 StringBuffer matchStr;
-                DBGLOG("%*sMatch (%s) to field %d for field %s", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source);
+                DBGLOG("%*sMatch (%s) to field %d for field %s (%x)", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source, destRecInfo.queryType(idx)->fieldType);
                 if (match.subTrans)
                     match.subTrans->doDescribe(indent+2);
             }
@@ -838,6 +1000,7 @@ private:
     {
         unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
         size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
         RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets);
         size32_t estimate = destRecInfo.getFixedSize();
         if (!estimate)
@@ -849,6 +1012,8 @@ private:
         for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
         {
             const RtlFieldInfo *field = destRecInfo.queryField(idx);
+            if (field->omitable() && destRecInfo.excluded(field, builder.getSelf(), destConditions))
+                continue;
             const RtlTypeInfo *type = field->type;
             const MatchInfo &match = matchInfo[idx];
             if (match.matchType == match_none || match.matchType==match_fail)
@@ -860,148 +1025,158 @@ private:
                 size_t sourceOffset = sourceRow.getOffset(matchField);
                 const byte *source = sourceRec + sourceOffset;
                 size_t copySize = sourceRow.getSize(matchField);
-                switch (match.matchType)
-                {
-                case match_perfect:
-                {
-                    // Look ahead for other perfect matches and combine the copies
-                    while (idx < destRecInfo.getNumFields()-1)
-                    {
-                        const MatchInfo &nextMatch = matchInfo[idx+1];
-                        if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
-                        {
-                            idx++;
-                            matchField++;
-                        }
-                        else
-                            break;
-                    }
-                    size_t copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
-                    builder.ensureCapacity(offset+copySize, field->name);
-                    memcpy(builder.getSelf()+offset, source, copySize);
-                    offset += copySize;
-                    break;
-                }
-                case match_truncate:
-                {
-                    assert(type->isFixedSize());
-                    size32_t copySize = type->getMinSize();
-                    builder.ensureCapacity(offset+copySize, field->name);
-                    memcpy(builder.getSelf()+offset, source, copySize);
-                    offset += copySize;
-                    break;
-                }
-                case match_extend:
+                if (copySize == 0)  // Field is missing because of an ifblock - use default value
                 {
-                    assert(type->isFixedSize());
-                    size32_t destSize = type->getMinSize();
-                    builder.ensureCapacity(offset+destSize, field->name);
-                    memcpy(builder.getSelf()+offset, source, copySize);
-                    offset += copySize;
-                    unsigned fillSize = destSize - copySize;
-                    memset(builder.getSelf()+offset, match.fillChar, fillSize);
-                    offset += fillSize;
-                    break;
+                    offset = type->buildNull(builder, offset, field);
                 }
-                case match_typecast:
-                    offset = translateScalar(builder, offset, field, type, sourceType, source);
-                    break;
-                case match_link:
+                else
                 {
-                    // a 32-bit record count, and a (linked) pointer to an array of record pointers
-                    byte *dest = builder.ensureCapacity(offset+sizeof(size32_t)+sizeof(const byte **), field->name)+offset;
-                    *(size32_t *)dest = *(size32_t *)source;
-                    *(const byte ***)(dest + sizeof(size32_t)) = rtlLinkRowset(*(const byte ***)(source + sizeof(size32_t)));
-                    offset += sizeof(size32_t)+sizeof(const byte **);
-                    break;
-                }
-                case match_recurse:
-                    if (type->getType()==type_record)
-                        offset = match.subTrans->doTranslate(builder, offset, source);
-                    else if (type->isLinkCounted())
+                    switch (match.matchType & ~match_inifblock)
                     {
-                        // a 32-bit record count, and a pointer to an array of record pointers
-                        IEngineRowAllocator *childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
-                        assertex(childAllocator);  // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
-
-                        size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
-                        builder.ensureCapacity(offset+sizeInBytes, field->name);
-                        size32_t numRows = 0;
-                        const byte **childRows = nullptr;
-                        if (sourceType->isLinkCounted())
-                        {
-                            // a 32-bit count, then a pointer to the source rows
-                            size32_t childCount = *(size32_t *) source;
-                            source += sizeof(size32_t);
-                            const byte ** sourceRows = *(const byte***) source;
-                            for (size32_t childRow = 0; childRow < childCount; childRow++)
-                            {
-                                RtlDynamicRowBuilder childBuilder(*childAllocator);
-                                size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, sourceRows[childRow]);
-                                childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
-                            }
-                        }
-                        else
+                    case match_perfect:
+                    {
+                        // Look ahead for other perfect matches and combine the copies
+                        while (idx < destRecInfo.getNumFields()-1)
                         {
-                            // a 32-bit size, then rows inline
-                            size32_t childSize = *(size32_t *) source;
-                            source += sizeof(size32_t);
-                            const byte *initialSource = source;
-                            while ((size_t)(source - initialSource) < childSize)
+                            const MatchInfo &nextMatch = matchInfo[idx+1];
+                            if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
                             {
-                                RtlDynamicRowBuilder childBuilder(*childAllocator);
-                                size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, source);
-                                childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
-                                source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
+                                idx++;
+                                matchField++;
                             }
+                            else
+                                break;
                         }
-                        // Go back in and patch the count, remembering it may have moved
-                        rtlWriteInt4(builder.getSelf()+offset, numRows);
-                        * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
-                        offset += sizeInBytes;
+                        size_t copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
+                        builder.ensureCapacity(offset+copySize, field->name);
+                        memcpy(builder.getSelf()+offset, source, copySize);
+                        offset += copySize;
+                        break;
                     }
-                    else
+                    case match_truncate:
+                    {
+                        assert(type->isFixedSize());
+                        size32_t copySize = type->getMinSize();
+                        builder.ensureCapacity(offset+copySize, field->name);
+                        memcpy(builder.getSelf()+offset, source, copySize);
+                        offset += copySize;
+                        break;
+                    }
+                    case match_extend:
                     {
-                        size32_t countOffset = offset;
-                        byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
-                        offset += sizeof(size32_t);
-                        size32_t initialOffset = offset;
-                        *(size32_t *)dest = 0;  // patched below when true figure known
-                        if (sourceType->isLinkCounted())
+                        assert(type->isFixedSize());
+                        size32_t destSize = type->getMinSize();
+                        builder.ensureCapacity(offset+destSize, field->name);
+                        memcpy(builder.getSelf()+offset, source, copySize);
+                        offset += copySize;
+                        unsigned fillSize = destSize - copySize;
+                        memset(builder.getSelf()+offset, match.fillChar, fillSize);
+                        offset += fillSize;
+                        break;
+                    }
+                    case match_typecast:
+                        offset = translateScalar(builder, offset, field, type, sourceType, source);
+                        break;
+                    case match_link:
+                    {
+                        // a 32-bit record count, and a (linked) pointer to an array of record pointers
+                        byte *dest = builder.ensureCapacity(offset+sizeof(size32_t)+sizeof(const byte **), field->name)+offset;
+                        *(size32_t *)dest = *(size32_t *)source;
+                        *(const byte ***)(dest + sizeof(size32_t)) = rtlLinkRowset(*(const byte ***)(source + sizeof(size32_t)));
+                        offset += sizeof(size32_t)+sizeof(const byte **);
+                        break;
+                    }
+                    case match_recurse:
+                        if (type->getType()==type_record)
+                            offset = match.subTrans->doTranslate(builder, offset, source);
+                        else if (type->isLinkCounted())
                         {
-                            // a 32-bit count, then a pointer to the source rows
-                            size32_t childCount = *(size32_t *) source;
-                            source += sizeof(size32_t);
-                            const byte ** sourceRows = *(const byte***) source;
-                            for (size32_t childRow = 0; childRow < childCount; childRow++)
+                            // a 32-bit record count, and a pointer to an array of record pointers
+                            IEngineRowAllocator *childAllocator = builder.queryAllocator()->createChildRowAllocator(type->queryChildType());
+                            assertex(childAllocator);  // May not be available when using serialized types (but unlikely to want to create linkcounted children remotely either)
+
+                            size32_t sizeInBytes = sizeof(size32_t) + sizeof(void *);
+                            builder.ensureCapacity(offset+sizeInBytes, field->name);
+                            size32_t numRows = 0;
+                            const byte **childRows = nullptr;
+                            if (sourceType->isLinkCounted())
+                            {
+                                // a 32-bit count, then a pointer to the source rows
+                                size32_t childCount = *(size32_t *) source;
+                                source += sizeof(size32_t);
+                                const byte ** sourceRows = *(const byte***) source;
+                                for (size32_t childRow = 0; childRow < childCount; childRow++)
+                                {
+                                    RtlDynamicRowBuilder childBuilder(*childAllocator);
+                                    size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, sourceRows[childRow]);
+                                    childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
+                                }
+                            }
+                            else
                             {
-                                offset = match.subTrans->doTranslate(builder, offset, sourceRows[childRow]);
+                                // a 32-bit size, then rows inline
+                                size32_t childSize = *(size32_t *) source;
+                                source += sizeof(size32_t);
+                                const byte *initialSource = source;
+                                while ((size_t)(source - initialSource) < childSize)
+                                {
+                                    RtlDynamicRowBuilder childBuilder(*childAllocator);
+                                    size32_t childLen = match.subTrans->doTranslate(childBuilder, 0, source);
+                                    childRows = childAllocator->appendRowOwn(childRows, ++numRows, (void *) childBuilder.finalizeRowClear(childLen));
+                                    source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
+                                }
                             }
+                            // Go back in and patch the count, remembering it may have moved
+                            rtlWriteInt4(builder.getSelf()+offset, numRows);
+                            * ( const void * * ) (builder.getSelf()+offset+sizeof(size32_t)) = childRows;
+                            offset += sizeInBytes;
                         }
                         else
                         {
-                            // a 32-bit size, then rows inline
-                            size32_t childSize = *(size32_t *) source;
-                            source += sizeof(size32_t);
-                            const byte *initialSource = source;
-                            while ((size_t)(source - initialSource) < childSize)
+                            size32_t countOffset = offset;
+                            byte *dest = builder.ensureCapacity(offset+sizeof(size32_t), field->name)+offset;
+                            offset += sizeof(size32_t);
+                            size32_t initialOffset = offset;
+                            *(size32_t *)dest = 0;  // patched below when true figure known
+                            if (sourceType->isLinkCounted())
                             {
-                                offset = match.subTrans->doTranslate(builder, offset, source);
-                                source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
+                                // a 32-bit count, then a pointer to the source rows
+                                size32_t childCount = *(size32_t *) source;
+                                source += sizeof(size32_t);
+                                const byte ** sourceRows = *(const byte***) source;
+                                for (size32_t childRow = 0; childRow < childCount; childRow++)
+                                {
+                                    offset = match.subTrans->doTranslate(builder, offset, sourceRows[childRow]);
+                                }
                             }
+                            else
+                            {
+                                // a 32-bit size, then rows inline
+                                size32_t childSize = *(size32_t *) source;
+                                source += sizeof(size32_t);
+                                const byte *initialSource = source;
+                                while ((size_t)(source - initialSource) < childSize)
+                                {
+                                    offset = match.subTrans->doTranslate(builder, offset, source);
+                                    source += sourceType->queryChildType()->size(source, nullptr); // MORE - shame to repeat a calculation that the translate above almost certainly just did
+                                }
+                            }
+                            dest = builder.getSelf() + countOffset;  // Note - may have been moved by reallocs since last calculated
+                            *(size32_t *)dest = offset - initialOffset;
                         }
-                        dest = builder.getSelf() + countOffset;  // Note - may have been moved by reallocs since last calculated
-                        *(size32_t *)dest = offset - initialOffset;
+                        break;
+                    default:
+                        throwUnexpected();
                     }
-                    break;
-                default:
-                    throwUnexpected();
                 }
             }
         }
         if (estimate && offset-origOffset != estimate)
         {
-            assert(offset-origOffset > estimate);  // Estimate is always supposed to be conservative
+            // Note - ifblocks make this assertion invalid. We do not account for potentially omitted fields
+            // when estimating target record size.
+            if (!destRecInfo.getNumIfBlocks())
+                assert(offset-origOffset > estimate);  // Estimate is always supposed to be conservative
 #ifdef TRACE_TRANSLATION
             DBGLOG("Wrote %u bytes to record (estimate was %u)\n", offset-origOffset, estimate);
 #endif
@@ -1179,6 +1354,7 @@ private:
                             break;
                         case type_row:      // These are not expected I think...
                             throwUnexpected();
+                        case type_ifblock:
                         case type_record:
                         case type_table:
                         {
@@ -1251,6 +1427,8 @@ private:
                 }
                 else
                     info.matchType = match_typecast;
+                if (sourceRecInfo.queryField(info.matchIdx)->flags & RFTMinifblock)
+                    info.matchType |= match_inifblock;  // Avoids incorrect commoning up of adjacent matches
                 // MORE - could note the highest interesting fieldnumber in the source and not bother filling in offsets after that
                 // Not sure it would help much though - usually need to know the total record size anyway in real life
                 if (idx != info.matchIdx)

rtl/eclrtl/rtldynfield.hpp

@@ -37,7 +37,7 @@ public:
     const RtlTypeInfo *childType = nullptr;
     const RtlFieldInfo * * fieldsArray = nullptr;
 
-    const RtlTypeInfo *createRtlTypeInfo() const;
+    const RtlTypeInfo *createRtlTypeInfo(IThorIndexCallback *_callback) const;
 };
 
 interface ITypeInfo;
@@ -111,11 +111,11 @@ interface IDynamicTransform : public IInterface
 
 extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo);
 
-extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer();
+extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback);
 
 extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t);
 
-extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t);
+extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t, bool useBias);
 
 /**
  * Serialize metadata of supplied record to JSON, and return it to ECL caller as a string. Used for testing serializer.

rtl/eclrtl/rtlfield.cpp

@@ -432,7 +432,7 @@ void RtlIntTypeInfo::getString(size32_t & resultLen, char * & result, const void
     if (isUnsigned())
         rtlUInt8ToStrX(resultLen, result,  rtlReadUInt(ptr, length));
     else
-        rtlInt8ToStrX(resultLen, result,  rtlReadInt(ptr, length));
+        rtlInt8ToStrX(resultLen, result, rtlReadInt(ptr, length));
 }
 
 void RtlIntTypeInfo::getUtf8(size32_t & resultLen, char * & result, const void * ptr) const
@@ -443,7 +443,7 @@ void RtlIntTypeInfo::getUtf8(size32_t & resultLen, char * & result, const void *
 __int64 RtlIntTypeInfo::getInt(const void * ptr) const
 {
     if (isUnsigned())
-         return rtlReadUInt(ptr, length);
+        return rtlReadUInt(ptr, length);
     else
         return rtlReadInt(ptr, length);
 }
@@ -494,6 +494,97 @@ bool RtlIntTypeInfo::canExtend(char &fillChar) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+size32_t RtlFileposTypeInfo::build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const
+{
+    throwUnexpected();  // This is only expected to be used for reading at present
+}
+
+size32_t RtlFileposTypeInfo::buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const
+{
+    throwUnexpected();  // This is only expected to be used for reading at present
+}
+
+size32_t RtlFileposTypeInfo::buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const
+{
+    throwUnexpected();  // This is only expected to be used for reading at present
+}
+
+size32_t RtlFileposTypeInfo::buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const
+{
+    throwUnexpected();  // This is only expected to be used for reading at present
+}
+
+size32_t RtlFileposTypeInfo::process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const
+{
+    assertex(callback);
+    if (isUnsigned())
+        target.processUInt(callback->getFilePosition(nullptr), field);
+    else
+        target.processInt(callback->getFilePosition(nullptr), field);
+    return length;
+}
+
+size32_t RtlFileposTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
+{
+    assertex(callback);
+    if (isUnsigned())
+        target.outputUInt(callback->getFilePosition(nullptr), length, queryScalarXPath(field));
+    else
+        target.outputInt(callback->getFilePosition(nullptr), length, queryScalarXPath(field));
+    return length;
+}
+
+void RtlFileposTypeInfo::getString(size32_t & resultLen, char * & result, const void * ptr) const
+{
+    assertex(callback);
+    if (isUnsigned())
+        rtlUInt8ToStrX(resultLen, result, callback->getFilePosition(nullptr));
+    else
+        rtlInt8ToStrX(resultLen, result, callback->getFilePosition(nullptr));
+}
+
+void RtlFileposTypeInfo::getUtf8(size32_t & resultLen, char * & result, const void * ptr) const
+{
+    getString(resultLen, result, ptr);
+}
+
+__int64 RtlFileposTypeInfo::getInt(const void * ptr) const
+{
+    assertex(callback);
+    return (__int64) callback->getFilePosition(nullptr);
+}
+
+double RtlFileposTypeInfo::getReal(const void * ptr) const
+{
+    assertex(callback);
+    if (isUnsigned())
+        return (double) (__uint64) callback->getFilePosition(nullptr);
+    else
+        return (double) (__int64) callback->getFilePosition(nullptr);
+}
+
+bool RtlFileposTypeInfo::canTruncate() const
+{
+    return false;
+}
+
+bool RtlFileposTypeInfo::canExtend(char &fillChar) const
+{
+    return false;
+}
+
+void RtlFileposTypeInfo::setCallback(IThorIndexCallback *_callback)
+{
+    callback = _callback;
+}
+
+int RtlFileposTypeInfo::compare(const byte * left, const byte * right) const
+{
+    throwUnexpected();  // Not needed and unimplementable
+}
+
+//-------------------------------------------------------------------------------------------------------------------
+
 size32_t RtlSwapIntTypeInfo::build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const
 {
     builder.ensureCapacity(length+offset, queryName(field));
@@ -567,7 +658,7 @@ double RtlSwapIntTypeInfo::getReal(const void * ptr) const
     if (isUnsigned())
         return (double) rtlReadSwapUInt(ptr, length);
     else
-        return (double) rtlReadSwapInt(ptr, length);
+        return (double) (rtlReadSwapInt(ptr, length));
 }
 
 int RtlSwapIntTypeInfo::compare(const byte * left, const byte * right) const
@@ -607,6 +698,116 @@ bool RtlSwapIntTypeInfo::canExtend(char &fillChar) const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+// Should be renamed to keyedint - unsigned as well as signed cases here.
+
+size32_t RtlKeyedIntTypeInfo::build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const
+{
+    __int64 val = isUnsigned() ? (__int64) source.getUnsignedResult(field) : source.getSignedResult(field);
+    return buildInt(builder, offset, field, val);
+}
+
+size32_t RtlKeyedIntTypeInfo::buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const
+{
+    builder.ensureCapacity(length+offset, queryName(field));
+    if (isUnsigned())
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+        rtlWriteSwapInt(builder.getSelf() + offset, val, length);
+#else
+        rtlWriteInt(builder.getSelf() + offset, val, length);
+#endif
+    else
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+        rtlWriteSwapInt(builder.getSelf() + offset, addBias(val, length), length);
+#else
+        rtlWriteInt(builder.getSelf() + offset, addBias(val, length), length);
+#endif
+    offset += length;
+    return offset;
+}
+
+size32_t RtlKeyedIntTypeInfo::buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const
+{
+    return buildInt(builder, offset, field, rtlStrToInt8(size, value));
+}
+
+size32_t RtlKeyedIntTypeInfo::buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const
+{
+    size32_t size = rtlUtf8Length(len, value);
+    return buildInt(builder, offset, field, rtlStrToInt8(size, value));
+}
+
+size32_t RtlKeyedIntTypeInfo::process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const
+{
+    if (isUnsigned())
+        target.processUInt(getUInt(self), field);
+    else
+        target.processInt(getInt(self), field);
+    return length;
+}
+
+size32_t RtlKeyedIntTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
+{
+    if (isUnsigned())
+        target.outputUInt(getUInt(self), length, queryScalarXPath(field));
+    else
+        target.outputInt(getInt(self), length, queryScalarXPath(field));
+    return length;
+}
+
+void RtlKeyedIntTypeInfo::getString(size32_t & resultLen, char * & result, const void * ptr) const
+{
+    if (isUnsigned())
+        rtlUInt8ToStrX(resultLen, result, getUInt(ptr));
+    else
+        rtlInt8ToStrX(resultLen, result, getInt(ptr));
+}
+
+void RtlKeyedIntTypeInfo::getUtf8(size32_t & resultLen, char * & result, const void * ptr) const
+{
+    getString(resultLen, result, ptr);
+}
+
+__int64 RtlKeyedIntTypeInfo::getInt(const void * ptr) const
+{
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+    if (isUnsigned())
+        return rtlReadSwapUInt(ptr, length);
+    else
+        return removeBias(rtlReadSwapUInt(ptr, length), length);
+#else
+    if (isUnsigned())
+        return rtlReadUInt(ptr, length);
+    else
+        return removeBias(rtlReadInt(ptr, length), length);
+#endif
+}
+
+double RtlKeyedIntTypeInfo::getReal(const void * ptr) const
+{
+    if (isUnsigned())
+        return (double) getUInt(ptr);
+    else
+        return (double) getInt(ptr);
+}
+
+int RtlKeyedIntTypeInfo::compare(const byte * left, const byte * right) const
+{
+    // The whole point of biased ints is that we can do this:
+    return memcmp(left, right, length);
+}
+
+unsigned __int64 RtlKeyedIntTypeInfo::addBias(__int64 value, unsigned length)
+{
+    return value + ((unsigned __int64)1 << (length*8-1));
+}
+
+__int64 RtlKeyedIntTypeInfo::removeBias(unsigned __int64 value, unsigned length)
+{
+    return value - ((unsigned __int64)1 << (length*8-1));
+}
+
+//-------------------------------------------------------------------------------------------------------------------
+
 size32_t RtlPackedIntTypeInfo::getMinSize() const
 {
     return 1;
@@ -3251,6 +3452,15 @@ int RtlIfBlockTypeInfo::compare(const byte * left, const byte * right) const
         return 0;
 }
 
+bool RtlDynamicIfBlockTypeInfo::getCondition(const byte * selfrow) const
+{
+#ifdef _DEBUG
+    // Temporary code to help my testing, until proper implementation available
+    return selfrow[3] != 2;
+#endif
+    UNIMPLEMENTED;
+}
+
 
 
 //-------------------------------------------------------------------------------------------------------------------

rtl/eclrtl/rtlfield.hpp

@@ -1,6 +1,6 @@
 /*##############################################################################
 
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -132,6 +132,37 @@ struct ECLRTL_API RtlIntTypeInfo : public RtlTypeInfoBase
     virtual int compare(const byte * left, const byte * right) const override;
 };
 
+struct ECLRTL_API RtlFileposTypeInfo : public RtlTypeInfoBase
+{
+    // Used for values stored in the fieldpos field of indexes
+    constexpr inline RtlFileposTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo *_child, IThorIndexCallback *_callback)
+    : RtlTypeInfoBase(_fieldType, _length), child(_child), callback(_callback)
+    {}
+    virtual void doDelete() const final override { delete this; }
+
+    virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
+
+    virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
+    virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
+    virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
+    virtual __int64 getInt(const void * ptr) const override;
+    virtual double getReal(const void * ptr) const override;
+    virtual bool canTruncate() const override;
+    virtual bool canExtend(char &fillChar) const override;
+    virtual bool isNumeric() const override { return true; }
+    virtual int compare(const byte * left, const byte * right) const override;
+    virtual const RtlTypeInfo * queryChildType() const override { return child; }
+
+    void setCallback(IThorIndexCallback *_callback);
+private:
+    const RtlTypeInfo *child = nullptr;
+    IThorIndexCallback *callback = nullptr;
+};
+
 struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase
 {
     constexpr inline RtlSwapIntTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
@@ -154,6 +185,31 @@ struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase
     virtual int compare(const byte * left, const byte * right) const override;
 };
 
+struct ECLRTL_API RtlKeyedIntTypeInfo final : public RtlTypeInfoBase
+{
+    constexpr inline RtlKeyedIntTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo *_child) : RtlTypeInfoBase(_fieldType, _length), child(_child) {}
+    virtual void doDelete() const final override { delete this; }
+
+    virtual size32_t build(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, IFieldSource &source) const override;
+    virtual size32_t buildInt(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, __int64 val) const override;
+    virtual size32_t buildString(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t size, const char *value) const override;
+    virtual size32_t buildUtf8(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, size32_t len, const char *value) const override;
+
+    virtual size32_t process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const override;
+    virtual size32_t toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const override;
+    virtual void getString(size32_t & resultLen, char * & result, const void * ptr) const override;
+    virtual void getUtf8(size32_t & resultLen, char * & result, const void * ptr) const override;
+    virtual __int64 getInt(const void * ptr) const override;
+    virtual double getReal(const void * ptr) const override;
+    virtual bool isNumeric() const override { return true; }
+    virtual int compare(const byte * left, const byte * right) const override;
+private:
+    inline __uint64 getUInt(const void * ptr) const { return (__uint64) getInt(ptr); }
+    static unsigned __int64 addBias(__int64 value, unsigned length);
+    static __int64 removeBias(unsigned __int64 value, unsigned length);
+    const RtlTypeInfo *child = nullptr;
+};
+
 struct ECLRTL_API RtlPackedIntTypeInfo : public RtlTypeInfoBase
 {
     constexpr inline RtlPackedIntTypeInfo(unsigned _fieldType, unsigned _length) : RtlTypeInfoBase(_fieldType, _length) {}
@@ -490,7 +546,7 @@ struct ECLRTL_API RtlDictionaryTypeInfo : public RtlCompoundTypeInfo
 
 struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
 {
-    constexpr inline RtlIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlTypeInfoBase(_fieldType|RFTMnoserialize, _length), fields(_fields) {}
+    constexpr inline RtlIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlTypeInfoBase(_fieldType, _length), fields(_fields) {}
     const RtlFieldInfo * const * fields;                // null terminated
 
     virtual bool getCondition(const byte * selfrow) const = 0;
@@ -511,6 +567,12 @@ struct ECLRTL_API RtlIfBlockTypeInfo : public RtlTypeInfoBase
     virtual const RtlFieldInfo * const * queryFields() const override { return fields; }
 };
 
+struct ECLRTL_API RtlDynamicIfBlockTypeInfo final : public RtlIfBlockTypeInfo
+{
+    constexpr inline RtlDynamicIfBlockTypeInfo(unsigned _fieldType, unsigned _length, const RtlFieldInfo * const * _fields) : RtlIfBlockTypeInfo(_fieldType, _length, _fields) {}
+    virtual bool getCondition(const byte * selfrow) const override;
+    virtual void doDelete() const final override { delete this; }
+};
 
 struct ECLRTL_API RtlBitfieldTypeInfo : public RtlTypeInfoBase
 {

rtl/eclrtl/rtlformat.hpp

@@ -7,8 +7,19 @@
 #include "eclrtl.hpp"
 #include "eclhelper.hpp"
 
+interface IXmlWriterExt : extends IXmlWriter
+{
+    virtual IXmlWriterExt & clear() = 0;
+    virtual size32_t length() const = 0;
+    virtual const char *str() const = 0;
+    virtual IInterface *saveLocation() const = 0;
+    virtual void rewindTo(IInterface *location) = 0;
+    virtual void cutFrom(IInterface *location, StringBuffer& databuf) = 0;
+    virtual void outputNumericString(const char *field, const char *fieldname) = 0;
+    virtual void outputInline(const char* text) = 0;
+};
 
-class ECLRTL_API SimpleOutputWriter : implements IXmlWriter, public CInterface
+class ECLRTL_API SimpleOutputWriter : implements IXmlWriterExt, public CInterface
 {
     void outputFieldSeparator();
     bool separatorNeeded;
@@ -16,35 +27,41 @@ public:
     SimpleOutputWriter();
     IMPLEMENT_IINTERFACE;
 
-    SimpleOutputWriter & clear();
-    unsigned length() const                                 { return out.length(); }
-    const char * str() const                                { return out.str(); }
+    virtual SimpleOutputWriter & clear() override;
+    virtual size32_t length() const override                { return out.length(); }
+    virtual const char * str() const override               { return out.str(); }
 
-    virtual void outputQuoted(const char *text);
-    virtual void outputQString(unsigned len, const char *field, const char *fieldname);
-    virtual void outputString(unsigned len, const char *field, const char *fieldname);
-    virtual void outputBool(bool field, const char *fieldname);
-    virtual void outputData(unsigned len, const void *field, const char *fieldname);
-    virtual void outputReal(double field, const char *fieldname);
-    virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
-    virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
-    virtual void outputUnicode(unsigned len, const UChar *field, const char *fieldname);
-    virtual void outputUtf8(unsigned len, const char *field, const char *fieldname);
-    virtual void outputBeginNested(const char *fieldname, bool nestChildren);
-    virtual void outputEndNested(const char *fieldname);
-    virtual void outputBeginDataset(const char *dsname, bool nestChildren){}
-    virtual void outputEndDataset(const char *dsname){}
-    virtual void outputBeginArray(const char *fieldname){}
-    virtual void outputEndArray(const char *fieldname){}
-    virtual void outputSetAll();
-    virtual void outputInlineXml(const char *text){} //for appending raw xml content
-    virtual void outputXmlns(const char *name, const char *uri){}
+    virtual void outputQuoted(const char *text) override;
+    virtual void outputQString(unsigned len, const char *field, const char *fieldname) override;
+    virtual void outputString(unsigned len, const char *field, const char *fieldname) override;
+    virtual void outputBool(bool field, const char *fieldname) override;
+    virtual void outputData(unsigned len, const void *field, const char *fieldname) override;
+    virtual void outputReal(double field, const char *fieldname) override;
+    virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname) override;
+    virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname) override;
+    virtual void outputUnicode(unsigned len, const UChar *field, const char *fieldname) override;
+    virtual void outputUtf8(unsigned len, const char *field, const char *fieldname) override;
+    virtual void outputBeginNested(const char *fieldname, bool nestChildren) override;
+    virtual void outputEndNested(const char *fieldname) override;
+    virtual void outputBeginDataset(const char *dsname, bool nestChildren) override {}
+    virtual void outputEndDataset(const char *dsname) override {}
+    virtual void outputBeginArray(const char *fieldname) override {}
+    virtual void outputEndArray(const char *fieldname) override {}
+    virtual void outputSetAll() override;
+    virtual void outputInlineXml(const char *text) override {} //for appending raw xml content
+    virtual void outputXmlns(const char *name, const char *uri) override {}
 
-    virtual void outputInt(__int64 field, unsigned size, const char *fieldname);
-    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname);
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname) override;
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname) override;
 
     void newline();
 
+    virtual IInterface *saveLocation() const override { UNIMPLEMENTED; }
+    virtual void rewindTo(IInterface *location) override { UNIMPLEMENTED; }
+    virtual void cutFrom(IInterface *location, StringBuffer& databuf) override { UNIMPLEMENTED; }
+    virtual void outputNumericString(const char *field, const char *fieldname) override { UNIMPLEMENTED; }
+    virtual void outputInline(const char* text) override { UNIMPLEMENTED; }
+
 protected:
     StringBuffer out;
 };
@@ -55,18 +72,6 @@ interface IXmlStreamFlusher
     virtual void flushXML(StringBuffer &current, bool isClose) = 0;
 };
 
-interface IXmlWriterExt : extends IXmlWriter
-{
-    virtual IXmlWriterExt & clear() = 0;
-    virtual size32_t length() const = 0;
-    virtual const char *str() const = 0;
-    virtual IInterface *saveLocation() const = 0;
-    virtual void rewindTo(IInterface *location) = 0;
-    virtual void cutFrom(IInterface *location, StringBuffer& databuf) = 0;
-    virtual void outputNumericString(const char *field, const char *fieldname) = 0;
-    virtual void outputInline(const char* text) = 0;
-};
-
 class ECLRTL_API CommonXmlPosition : public CInterface, implements IInterface
 {
 public:

rtl/eclrtl/rtlrecord.cpp

@@ -64,10 +64,14 @@
  *    the last bitfield in a bitfield container has the size of the container, the others have 0 size.
  *
  * c. ifblocks
- *    Nasty.  Allowing direct access means the flag would need checking, and a field would need a pointer
- *    to its containing ifblock.  Cleanest to have a different derived implementation which was used if the record
- *    contained ifblocks which added calls to check ifblocks before size()/getValue() etc.
- *    Will require an extra row parameter to every RtlTypeInfo::getX() function.
+ *    A field in an ifblock can be viewed as a variable size field - size is either 0 or normal size
+ *    depending on the condition in question. We add a flag and a pointer to the ifblock info into
+ *    each contained field. Only properly supported in expanded mode.
+ *    Might be cleanest to have a different derived implementation which was used if the record
+ *    contained ifblocks, though this would mean we would need to make some functions virtual or else move
+ *    the responsibility for knowing about ifblocks out to all customers?
+ *    Added calls to check ifblocks before size()/getValue() etc. will require an extra row parameter
+ *    to every RtlTypeInfo::getX() function? Not if it is required that the offsets have been set first.
  *    Evaluating the test expression without compiling will require expression interpreting.
  *
  * d. alien datatypes
@@ -84,18 +88,20 @@
  *   For nested selects the code would need to be consistent.
  */
 
-static unsigned countFields(const RtlFieldInfo * const * fields, bool & containsNested)
+static unsigned countFields(const RtlFieldInfo * const * fields, bool & containsNested, unsigned &numIfBlocks)
 {
     unsigned cnt = 0;
     for (;*fields;fields++)
     {
         const RtlTypeInfo * type = (*fields)->type;
-        if (type->getType() == type_record)
+        if (type->getType() == type_record || type->getType()==type_ifblock)
         {
             containsNested = true;
+            if (type->getType()==type_ifblock)
+                numIfBlocks++;
             const RtlFieldInfo * const * nested = type->queryFields();
             if (nested)
-                cnt += countFields(nested, containsNested);
+                cnt += countFields(nested, containsNested, numIfBlocks);
         }
         else
             cnt++;
@@ -103,21 +109,68 @@ static unsigned countFields(const RtlFieldInfo * const * fields, bool & contains
     return cnt;
 }
 
+class IfBlockInfo
+{
+public:
+    IfBlockInfo(const RtlFieldInfo &_field, const IfBlockInfo *_parent, unsigned _idx)
+    : field(_field), parent(_parent), idx(_idx)
+    {}
+
+    bool excluded(const byte *row, byte *conditions) const
+    {
+        if (conditions[idx]==2)
+        {
+            if (parent && parent->excluded(row, conditions))
+                conditions[idx] = 0;
+            else
+            {
+                const RtlIfBlockTypeInfo *cond = static_cast<const RtlIfBlockTypeInfo *>(field.type);
+                conditions[idx] = cond->getCondition(row) ? 1 : 0;
+            }
+        }
+        return conditions[idx]==0;
+    }
+private:
+    const RtlFieldInfo &field;
+    const IfBlockInfo *parent = nullptr;  // for nested ifblocks
+    unsigned idx;
+};
+
+class RtlCondFieldStrInfo : public RtlFieldStrInfo
+{
+public:
+    RtlCondFieldStrInfo(const RtlFieldInfo &from, const IfBlockInfo &_ifblock)
+    : RtlFieldStrInfo(from.name, from.xpath, from.type, from.flags | (RFTMinifblock|RFTMdynamic), (const char *) from.initializer),
+      origField(from),ifblock(_ifblock)
+    {
+    }
+public:
+    const RtlFieldInfo &origField;
+    const IfBlockInfo &ifblock;
+};
 
-static unsigned expandNestedRows(unsigned idx, const char *prefix, const RtlFieldInfo * const * fields, const RtlFieldInfo * * target, const char * *names)
+static unsigned expandNestedRows(unsigned idx, const char *prefix, const RtlFieldInfo * const * fields, const RtlFieldInfo * * target, const char * *names, const IfBlockInfo *inIfBlock, ConstPointerArrayOf<IfBlockInfo> &ifblocks)
 {
     for (;*fields;fields++)
     {
         const RtlFieldInfo * cur = *fields;
         const RtlTypeInfo * type = cur->type;
-        if (type->getType() == type_record)
+        bool isIfBlock  = type->getType()==type_ifblock;
+        if (isIfBlock || type->getType() == type_record)
         {
+            const IfBlockInfo *nestIfBlock = inIfBlock;
+            if (isIfBlock)
+            {
+                nestIfBlock = new IfBlockInfo(*cur, inIfBlock, ifblocks.ordinality());
+                ifblocks.append(inIfBlock);
+            }
             const RtlFieldInfo * const * nested = type->queryFields();
             if (nested)
             {
                 StringBuffer newPrefix(prefix);
-                newPrefix.append(cur->name).append('.');
-                idx = expandNestedRows(idx, newPrefix.str(), nested, target, names);
+                if (cur->name && *cur->name)
+                    newPrefix.append(cur->name).append('.');
+                idx = expandNestedRows(idx, newPrefix.str(), nested, target, names, nestIfBlock, ifblocks);
             }
         }
         else
@@ -130,7 +183,13 @@ static unsigned expandNestedRows(unsigned idx, const char *prefix, const RtlFiel
             }
             else
                 names[idx] = nullptr;
-            target[idx++] = cur;
+            if (inIfBlock && !(cur->flags & RFTMinifblock))
+                target[idx++] = new RtlCondFieldStrInfo(*cur, *inIfBlock);
+            else
+            {
+                dbgassertex((cur->flags & RFTMdynamic) == 0);
+                target[idx++] = cur;
+            }
         }
     }
     return idx;
@@ -163,20 +222,23 @@ RtlRecord::RtlRecord(const RtlRecordTypeInfo & record, bool expandFields)
 
 RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : fields(_fields), originalFields(_fields), names(nullptr), nameMap(nullptr)
 {
-    //MORE: Does not cope with ifblocks.
     numVarFields = 0;
     numTables = 0;
+    numIfBlocks = 0;
+    ifblocks = nullptr;
     //Optionally expand out nested rows.
     if (expandFields)
     {
         bool containsNested = false;
-        numFields = countFields(fields, containsNested);
+        numFields = countFields(fields, containsNested, numIfBlocks);
         if (containsNested)
         {
+            ConstPointerArrayOf<IfBlockInfo> _ifblocks;
             const RtlFieldInfo * * allocated  = new const RtlFieldInfo * [numFields+1];
             names = new const char *[numFields];
             fields = allocated;
-            unsigned idx = expandNestedRows(0, nullptr, originalFields, allocated, names);
+            unsigned idx = expandNestedRows(0, nullptr, originalFields, allocated, names, nullptr, _ifblocks);
+            ifblocks = _ifblocks.detach();
             assertex(idx == numFields);
             allocated[idx] = nullptr;
         }
@@ -188,7 +250,7 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
     for (unsigned i=0; i < numFields; i++)
     {
         const RtlTypeInfo *curType = queryType(i);
-        if (!curType->isFixedSize())
+        if (!curType->isFixedSize() || (fields[i]->flags & RFTMinifblock))
             numVarFields++;
         if (curType->getType()==type_table || curType->getType()==type_record)
             numTables++;
@@ -218,7 +280,7 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
             break;
 
         const RtlTypeInfo * curType = queryType(i);
-        if (curType->isFixedSize())
+        if (curType->isFixedSize() && !(fields[i]->flags & RFTMinifblock))
         {
             size_t thisSize = curType->size(nullptr, nullptr);
             fixedOffset += thisSize;
@@ -256,8 +318,22 @@ RtlRecord::~RtlRecord()
     }
     if (fields != originalFields)
     {
+        for (const RtlFieldInfo * const * finger = fields; *finger; finger++)
+        {
+            const RtlFieldInfo *thisField = *finger;
+            if (thisField->flags & RFTMdynamic)
+                delete thisField;
+        }
         delete [] fields;
     }
+    if (ifblocks)
+    {
+        for (unsigned i = 0; i < numIfBlocks; i++)
+        {
+            delete(ifblocks[i]);
+        }
+        delete [] ifblocks;
+    }
     delete [] fixedOffsets;
     delete [] whichVariableOffset;
     delete [] variableFieldIds;
@@ -276,14 +352,39 @@ void RtlRecord::calcRowOffsets(size_t * variableOffsets, const void * _row, unsi
 {
     const byte * row = static_cast<const byte *>(_row);
     unsigned maxVarField = (numFieldsUsed>=numFields) ? numVarFields : whichVariableOffset[numFieldsUsed];
-    size32_t varoffset = 0;
-    for (unsigned i = 0; i < maxVarField; i++)
+    if (numIfBlocks)
     {
-        unsigned fieldIndex = variableFieldIds[i];
-        size32_t offset = fixedOffsets[fieldIndex] + varoffset;
-        size32_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
-        varoffset = offset+fieldSize;
-        variableOffsets[i+1] = varoffset;
+        byte *conditions = (byte *) alloca(numIfBlocks * sizeof(byte));
+        memset(conditions, 2, numIfBlocks);    // Meaning condition not yet calculated
+        for (unsigned i = 0; i < maxVarField; i++)
+        {
+            unsigned fieldIndex = variableFieldIds[i];
+            const RtlFieldInfo *field = fields[fieldIndex];
+            size_t offset = getOffset(variableOffsets, fieldIndex);
+            if (field->flags & RFTMinifblock)
+            {
+                const RtlCondFieldStrInfo *condfield = static_cast<const RtlCondFieldStrInfo *>(field);
+                if (condfield->ifblock.excluded(row, conditions))
+                {
+                    variableOffsets[i+1] = offset; // (meaning size ends up as zero);
+                    continue;
+                }
+            }
+            size_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
+            variableOffsets[i+1] = offset+fieldSize;
+        }
+    }
+    else
+    {
+        size32_t varoffset = 0;
+        for (unsigned i = 0; i < maxVarField; i++)
+        {
+            unsigned fieldIndex = variableFieldIds[i];
+            size32_t offset = fixedOffsets[fieldIndex] + varoffset;
+            size32_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
+            varoffset = offset+fieldSize;
+            variableOffsets[i+1] = varoffset;
+        }
     }
 #ifdef _DEBUG
     for (unsigned i = maxVarField; i < numVarFields; i++)
@@ -300,20 +401,28 @@ size32_t RtlRecord::getMinRecordSize() const
 
     size32_t minSize = 0;
     for (unsigned i=0; i < numFields; i++)
-        minSize += queryType(i)->getMinSize();
-
+    {
+        const RtlFieldInfo *field = fields[i];
+        if (!(field->flags & RFTMinifblock))
+            minSize += queryType(i)->getMinSize();
+    }
     return minSize;
 }
 
 size32_t RtlRecord::deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource & in) const
 {
     size32_t offset = 0;
+    byte *conditionValues = (byte *) alloca(numIfBlocks);
+    memset(conditionValues, 2, numIfBlocks);
     for (unsigned i = 0; i < numVarFields; i++)
     {
         unsigned fieldIndex = variableFieldIds[i];
+        const RtlFieldInfo *field = fields[fieldIndex];
         size32_t fixedSize = fixedOffsets[fieldIndex];
-        byte * self = rowBuilder.ensureCapacity(offset + fixedSize, "");
+        byte * self = rowBuilder.ensureCapacity(offset + fixedSize, ""); // Why not field->name?
         in.read(fixedSize, self + offset);
+        if (field->omitable() && static_cast<const RtlCondFieldStrInfo *>(field)->ifblock.excluded(self, conditionValues))
+            continue;
         offset = queryType(fieldIndex)->deserialize(rowBuilder, in, offset);
     }
     size32_t lastFixedSize = fixedOffsets[numFields];
@@ -324,6 +433,8 @@ size32_t RtlRecord::deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource
 
 void RtlRecord::readAhead(IRowDeserializerSource & in) const
 {
+    // Note - this should not and can not be used when ifblocks present - canprefetch flag should take care of that.
+    dbgassertex(numIfBlocks==0);
     for (unsigned i=0; i < numVarFields; i++)
     {
         unsigned fieldIndex = variableFieldIds[i];
@@ -375,6 +486,21 @@ const RtlRecord *RtlRecord::queryNested(unsigned fieldId) const
     return nullptr;
 }
 
+const RtlFieldInfo *RtlRecord::queryOriginalField(unsigned idx) const
+{
+    const RtlFieldInfo *field = queryField(idx);
+    if (field->flags & RFTMdynamic)
+        return &static_cast<const RtlCondFieldStrInfo *>(field)->origField;
+    else
+        return field;
+}
+
+
+bool RtlRecord::excluded(const RtlFieldInfo *field, const byte *row, byte *conditionValues)
+{
+    return (field->omitable() && static_cast<const RtlCondFieldStrInfo *>(field)->ifblock.excluded(row, conditionValues));
+}
+
 size32_t RtlRecord::getRecordSize(const void *_row) const
 {
     size32_t size = getFixedSize();
@@ -407,29 +533,59 @@ RtlRow::RtlRow(const RtlRecord & _info, const void * optRow, unsigned numOffsets
 __int64 RtlRow::getInt(unsigned field) const
 {
     const byte * self = reinterpret_cast<const byte *>(row);
-    const RtlTypeInfo * type = info.queryType(field);
-    return type->getInt(self + getOffset(field));
+    const RtlFieldInfo *fieldInfo = info.queryField(field);
+    const RtlTypeInfo * type = fieldInfo->type;
+    if (!fieldInfo->omitable() || getSize(field))
+        return type->getInt(self + getOffset(field));
+    else if (fieldInfo->initializer)
+        return type->getInt(fieldInfo->initializer);
+    else
+        return 0;
 }
 
 double RtlRow::getReal(unsigned field) const
 {
     const byte * self = reinterpret_cast<const byte *>(row);
-    const RtlTypeInfo * type = info.queryType(field);
-    return type->getReal(self + getOffset(field));
+    const RtlFieldInfo *fieldInfo = info.queryField(field);
+    const RtlTypeInfo * type = fieldInfo->type;
+    if (!fieldInfo->omitable() || getSize(field))
+        return type->getReal(self + getOffset(field));
+    else if (fieldInfo->initializer)
+        return type->getInt(fieldInfo->initializer);
+    else
+        return 0;
 }
 
 void RtlRow::getString(size32_t & resultLen, char * & result, unsigned field) const
 {
     const byte * self = reinterpret_cast<const byte *>(row);
-    const RtlTypeInfo * type = info.queryType(field);
-    return type->getString(resultLen, result, self + getOffset(field));
+    const RtlFieldInfo *fieldInfo = info.queryField(field);
+    const RtlTypeInfo * type = fieldInfo->type;
+    if (!fieldInfo->omitable() || getSize(field))
+        type->getString(resultLen, result, self + getOffset(field));
+    else if (fieldInfo->initializer)
+        type->getString(resultLen, result, fieldInfo->initializer);
+    else
+    {
+        resultLen = 0;
+        result = nullptr;
+    }
 }
 
 void RtlRow::getUtf8(size32_t & resultLen, char * & result, unsigned field) const
 {
     const byte * self = reinterpret_cast<const byte *>(row);
-    const RtlTypeInfo * type = info.queryType(field);
-    return type->getUtf8(resultLen, result, self + getOffset(field));
+    const RtlFieldInfo *fieldInfo = info.queryField(field);
+    const RtlTypeInfo * type = fieldInfo->type;
+    if (!fieldInfo->omitable() || getSize(field))
+        type->getUtf8(resultLen, result, self + getOffset(field));
+    else if (fieldInfo->initializer)
+        type->getUtf8(resultLen, result, fieldInfo->initializer);
+    else
+    {
+        resultLen = 0;
+        result = nullptr;
+    }
 }
 
 void RtlRow::setRow(const void * _row)

rtl/eclrtl/rtlrecord.hpp

@@ -192,6 +192,8 @@ protected:
 
 class FieldNameToFieldNumMap;
 
+class IfBlockInfo;
+
 class ECLRTL_API RtlRecord
 {
 public:
@@ -212,7 +214,6 @@ public:
         return fixedOffsets[field] + variableOffsets[whichVariableOffset[field]];
     }
 
-
     size_t getRecordSize(size_t * variableOffsets) const
     {
         return getOffset(variableOffsets, numFields);
@@ -225,11 +226,14 @@ public:
 
     inline unsigned getNumFields() const { return numFields; }
     inline unsigned getNumVarFields() const { return numVarFields; }
+    inline unsigned getNumIfBlocks() const { return numIfBlocks > 0; }
     inline const RtlFieldInfo * queryField(unsigned field) const { return fields[field]; }
+    const RtlFieldInfo * queryOriginalField(unsigned field) const;
     inline const RtlTypeInfo * queryType(unsigned field) const { return fields[field]->type; }
     const char * queryName(unsigned field) const;
     unsigned getFieldNum(const char *fieldName) const;
     const RtlRecord *queryNested(unsigned field) const;
+    static bool excluded(const RtlFieldInfo *field, const byte *row, byte *conditions);
 protected:
     size_t * fixedOffsets;         // fixed portion of the field offsets + 1 extra
     unsigned * whichVariableOffset;// which variable offset should be added to the fixed
@@ -238,10 +242,12 @@ protected:
     unsigned numFields;
     unsigned numVarFields;
     unsigned numTables;
+    unsigned numIfBlocks;
     const RtlFieldInfo * const * fields;
     const RtlFieldInfo * const * originalFields;
     const RtlRecord **nestedTables;
     const char **names;
+    const IfBlockInfo **ifblocks;
     mutable const FieldNameToFieldNumMap *nameMap;
 };
 

rtl/include/eclhelper.hpp

@@ -333,8 +333,11 @@ enum RtlFieldTypeMask
     RFTMunknownsize         = 0x00000400,                   // if set, the field is unknown size - and length is the maximum length
 
     RFTMalien               = 0x00000800,                   // this is the physical format of a user defined type, if unknown size we can't calculate it
-    RFTMcontainsifblock     = 0x00000800,                   // contains an if block - if set on a record then it contains ifblocks, so can't work out field offsets.
     RFTMhasnonscalarxpath   = 0x00001000,                   // field xpath contains multiple node, and is not therefore usable for naming scalar fields
+    RFTMbiased              = 0x00002000,                   // type is stored with a bias
+
+    RFTMinifblock           = 0x00004000,                   // on fields only, field is inside an ifblock (not presently generated)
+    RFTMdynamic             = 0x00008000,                   // Reserved for use by RtlRecord to indicate structure needs to be deleted
 
     // These flags are used in the serialized info only
     RFTMserializerFlags     = 0x01f00000,                   // Mask to remove from saved values
@@ -348,7 +351,7 @@ enum RtlFieldTypeMask
     RFTMcontainsunknown     = 0x10000000,                   // contains a field of unknown type that we can't process properly
     RFTMinvalidxml          = 0x20000000,                   // cannot be called to generate xml
     RFTMhasxmlattr          = 0x40000000,                   // if specified, then xml output includes an attribute (recursive)
-    RFTMnoserialize         = 0x80000000,                   // cannot serialize this typeinfo structure (contains ifblocks, dictionaries or other nasties)
+    RFTMnoserialize         = 0x80000000,                   // cannot serialize this typeinfo structure (contains dictionaries or other nasties)
 
     RFTMinherited           = (RFTMnoprefetch|RFTMcontainsunknown|RFTMinvalidxml|RFTMhasxmlattr|RFTMnoserialize)    // These flags are recursively set on any parent records too
 };
@@ -399,6 +402,7 @@ struct RtlTypeInfo : public RtlITypeInfo
     inline bool isEbcdic() const { return (fieldType & RFTMebcdic) != 0; }
     inline bool isFixedSize() const { return (fieldType & RFTMunknownsize) == 0; }
     inline bool isLinkCounted() const { return (fieldType & RFTMlinkcounted) != 0; }
+    inline bool isSigned() const { return (fieldType & RFTMunsigned) == 0; }
     inline bool isUnsigned() const { return (fieldType & RFTMunsigned) != 0; }
     inline unsigned getDecimalDigits() const { return (length & 0xffff); }
     inline unsigned getDecimalPrecision() const { return (length >> 16); }
@@ -435,9 +439,13 @@ struct RtlFieldInfo
 
     inline bool hasNonScalarXpath() const { return (flags & RFTMhasnonscalarxpath) != 0; }
 
+    inline bool omitable() const
+    {
+        return (flags & RFTMinifblock)!=0;
+    }
     inline bool isFixedSize() const 
     { 
-        return type->isFixedSize(); 
+        return type->isFixedSize();
     }
     inline size32_t size(const byte * self, const byte * selfrow) const 
     { 

system/include/rtlconst.hpp

@@ -31,10 +31,10 @@ enum type_vals
     type_string         = 4,
     type_alias          = 5, // This is only used when serializing expression graphs
     type_date           = 6,
-type_unused2            = 7,
-type_unused3            = 8,
+    type_swapfilepos    = 7,
+    type_biasedswapint  = 8,
     type_bitfield       = 9,
-type_unused4            = 10,
+    type_keyedint       = 10,
     type_char           = 11,
     type_enumerated     = 12,
     type_record         = 13,
@@ -53,7 +53,7 @@ type_unused4            = 10,
     type_swapint        = 26,
     type_none           = 27,
     type_packedint      = 28,
-type_unused5            = 29,
+    type_filepos        = 29,
     type_qstring        = 30,
     type_unicode        = 31,
     type_any            = 32,

system/jhtree/jhtree.cpp

@@ -109,7 +109,8 @@ size32_t SegMonitorList::getSize() const
 
 void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname)
 {
-    if (getSize() != keyedSize)
+    size32_t segSize = getSize();
+    if (segSize > keyedSize)
     {
         StringBuffer err;
         err.appendf("Key size mismatch on key %s - key size is %u, expected %u", keyname, keyedSize, getSize());
@@ -117,6 +118,8 @@ void SegMonitorList::checkSize(size32_t keyedSize, char const * keyname)
         EXCLOG(e, err.str());
         throw e;
     }
+    else if (segSize < keyedSize)
+        segMonitors.append(*createWildKeySegmentMonitor(segSize, keyedSize-segSize));
 }
 
 void SegMonitorList::setLow(unsigned segno, void *keyBuffer) const
@@ -706,6 +709,16 @@ public:
             return reinterpret_cast<byte const *>(keyBuffer);
     }
 
+    inline size32_t queryRowSize()
+    {
+        return keyCursor ? keyCursor->getSize() : 0;
+    }
+
+    inline unsigned __int64 querySequence()
+    {
+        return keyCursor ? keyCursor->getSequence() : 0;
+    }
+
     inline offset_t queryFpos()
     {
         return lookupFpos;
@@ -737,11 +750,14 @@ public:
                 lookupFpos = keyCursor->getFPos();
                 unsigned i = 0;
                 matched = true;
-                for (; i <= lastSeg; i++)
+                if (segs.segMonitors.length())
                 {
-                    matched = segs.segMonitors.item(i).matchesBuffer(keyBuffer);
-                    if (!matched)
-                        break;
+                    for (; i <= lastSeg; i++)
+                    {
+                        matched = segs.segMonitors.item(i).matchesBuffer(keyBuffer);
+                        if (!matched)
+                            break;
+                    }
                 }
                 if (matched)
                 {
@@ -3178,7 +3194,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
                 }
             }
         }
-        builder->finish();
+        builder->finish(nullptr, nullptr);
         out->flush();
     }
 

system/jhtree/jhtree.hpp

@@ -198,7 +198,9 @@ interface IKeyManager : public IInterface, extends IIndexReadContext
 
     virtual const byte *queryKeyBuffer(offset_t & fpos) = 0; //if using RLT: fpos is the translated value, so correct in a normal row
     virtual offset_t queryFpos() = 0; //if using RLT: this is the untranslated fpos, so correct as the part number in TLK but not in a normal row
-    virtual unsigned queryRecordSize() = 0;
+    virtual unsigned __int64 querySequence() = 0;
+    virtual size32_t queryRowSize() = 0;     // Size of current row as returned by queryKeyBuffer()
+    virtual unsigned queryRecordSize() = 0;  // Max size
 
     virtual bool lookup(bool exact) = 0;
     virtual unsigned __int64 getCount() = 0;

system/jhtree/keybuild.cpp

@@ -348,11 +348,6 @@ public:
     }
 
 public:
-    void finish(unsigned *fileCrc)
-    {
-        finish(NULL, fileCrc);
-    }
-
     void finish(IPropertyTree * metadata, unsigned * fileCrc)
     {
         if (NULL != activeNode)

system/jhtree/keybuild.hpp

@@ -94,8 +94,7 @@ typedef IArrayOf<CNodeInfo> NodeInfoArray;
 
 interface IKeyBuilder : public IInterface
 {
-    virtual void finish(unsigned *crc = NULL) = 0;
-    virtual void finish(IPropertyTree * metadata, unsigned * crc = NULL) = 0;
+    virtual void finish(IPropertyTree * metadata, unsigned * crc) = 0;
     virtual void processKeyData(const char *keyData, offset_t pos, size32_t recsize) = 0;
     virtual void addLeafInfo(CNodeInfo *info) = 0;
     virtual unsigned __int64 createBlob(size32_t size, const char * _ptr) = 0;

system/jhtree/keydiff.cpp

@@ -423,7 +423,7 @@ public:
     ~CKeyWriter()
     {
         if (keyBuilder)
-            keyBuilder->finish();
+            keyBuilder->finish(nullptr, nullptr);
     }
 
     void put(RowBuffer & buffer)

system/jlib/jarray.hpp

@@ -190,6 +190,12 @@ public:
         assertex(pos <= SELF::used);
         return &head[pos];
     }
+    MEMBER * detach()
+    {
+        MEMBER * head = (MEMBER *)SELF::_head;
+        SELF::_init();
+        return head;
+    }
     void sort(CompareFunc cf)
     {
         SELF::_doSort(sizeof(MEMBER), (StdCompare)cf);

system/jlib/jiface.hpp

@@ -270,10 +270,20 @@ CLASS::beforeDispose()
 
 typedef class CInterface * CInterfacePtr;
 
+// Two versions of this macro - with and without override
+// until such time as we have completely switched over to using override consistently
+// otherwise it's hard to use the override consistency check error effectively.
+
 #define IMPLEMENT_IINTERFACE_USING(x)                                                   \
     virtual void Link(void) const       { x::Link(); }                     \
     virtual bool Release(void) const    { return x::Release(); }
 
 #define IMPLEMENT_IINTERFACE    IMPLEMENT_IINTERFACE_USING(CInterface)
 
+#define IMPLEMENT_IINTERFACE_O_USING(x)                                                   \
+    virtual void Link(void) const override { x::Link(); }                     \
+    virtual bool Release(void) const override    { return x::Release(); }
+
+#define IMPLEMENT_IINTERFACE_O    IMPLEMENT_IINTERFACE_O_USING(CInterface)
+
 #endif

system/jlib/jlib.hpp

@@ -93,6 +93,7 @@ public:
     inline TYPE & tos(void) const                 { return (TYPE &)CIArray::tos(); }
     inline TYPE & tos(aindex_t num) const         { return (TYPE &)CIArray::tos(num); }
     inline TYPE **getArray(aindex_t pos = 0)      { return (TYPE **)CIArray::getArray(pos); }
+    inline TYPE **detach()                        { return (TYPE **)CIArray::detach(); }
     inline void append(TYPE& obj)                 { assert(&obj); CIArray::append(obj); } 
     inline void appendUniq(TYPE& obj)             { assert(&obj); CIArray::appendUniq(obj); } 
     inline void add(TYPE& obj, aindex_t pos)      { assert(&obj); CIArray::add(obj, pos); } 
@@ -110,6 +111,7 @@ public:
     inline TYPE & tos(void) const                 { return (TYPE &)CICopyArray::tos(); }
     inline TYPE & tos(aindex_t num) const         { return (TYPE &)CICopyArray::tos(num); }
     inline TYPE **getArray(aindex_t pos = 0)      { return (TYPE **)CICopyArray::getArray(pos); }
+    inline TYPE **detach()                        { return (TYPE **)CICopyArray::detach(); }
     inline void append(TYPE& obj)                 { assert(&obj); CICopyArray::append(obj); } 
     inline void appendUniq(TYPE& obj)             { assert(&obj); CICopyArray::appendUniq(obj); } 
     inline void add(TYPE& obj, aindex_t pos)      { assert(&obj); CICopyArray::add(obj, pos); } 
@@ -127,6 +129,7 @@ public:
     inline TYPE & tos(void) const                 { return (TYPE &)IArray::tos(); }
     inline TYPE & tos(aindex_t num) const         { return (TYPE &)IArray::tos(num); }
     inline TYPE **getArray(aindex_t pos = 0)      { return (TYPE **)IArray::getArray(pos); }
+    inline TYPE **detach()                        { return (TYPE **)IArray::detach(); }
     inline void append(TYPE& obj)                 { assert(&obj); IArray::append(obj); } 
     inline void appendUniq(TYPE& obj)             { assert(&obj); IArray::appendUniq(obj); } 
     inline void add(TYPE& obj, aindex_t pos)      { assert(&obj); IArray::add(obj, pos); } 
@@ -160,6 +163,7 @@ public:
     inline TYPE & tos(void) const                 { return (TYPE &)ICopyArray::tos(); }
     inline TYPE & tos(aindex_t num) const         { return (TYPE &)ICopyArray::tos(num); }
     inline TYPE **getArray(aindex_t pos = 0)      { return (TYPE **)ICopyArray::getArray(pos); }
+    inline TYPE **detach()                        { return (TYPE **)ICopyArray::detach(); }
     inline void append(TYPE& obj)                 { assert(&obj); ICopyArray::append(obj); } 
     inline void appendUniq(TYPE& obj)             { assert(&obj); ICopyArray::appendUniq(obj); } 
     inline void add(TYPE& obj, aindex_t pos)      { assert(&obj); ICopyArray::add(obj, pos); } 
@@ -177,6 +181,7 @@ public:
     inline TYPE * tos(void) const                 { return (TYPE *)IPointerArray::tos(); }
     inline TYPE * tos(aindex_t num) const         { return (TYPE *)IPointerArray::tos(num); }
     inline TYPE **getArray(aindex_t pos = 0)      { return (TYPE **)IPointerArray::getArray(pos); }
+    inline TYPE **detach()                        { return (TYPE **)IPointerArray::detach(); }
     inline void append(TYPE * obj)                { IPointerArray::append(obj); } 
     inline void appendUniq(TYPE * obj)            { IPointerArray::appendUniq(obj); } 
     inline void add(TYPE * obj, aindex_t pos)     { IPointerArray::add(obj, pos); } 
@@ -193,9 +198,10 @@ public:
     inline void add(TYPE * x, aindex_t pos)     { PointerArray::add(x, pos); }
     inline void append(TYPE * x)                { PointerArray::append(x); }
     inline aindex_t bAdd(TYPE * & newItem, PointerOfCompareFunc f, bool & isNew) { return PointerArray::bAdd(*(void * *)&newItem, (CompareFunc)f, isNew); }
-    inline aindex_t bSearch(const TYPE * & key, CompareFunc f) const    { return PointerArray:: bSearch(*(const void * *)&key, f); }
+    inline aindex_t bSearch(const TYPE * & key, PointerOfCompareFunc f) const    { return PointerArray:: bSearch(*(const void * *)&key, (CompareFunc)f); }
     inline aindex_t find(TYPE * x) const        { return PointerArray::find(x); }
     inline TYPE **getArray(aindex_t pos = 0)    { return (TYPE **)PointerArray::getArray(pos); }
+    inline TYPE **detach()                      { return (TYPE **)PointerArray::detach(); }
     inline TYPE * item(aindex_t pos) const      { return (TYPE *)PointerArray::item(pos); }
     inline TYPE * popGet()                      { return (TYPE *)PointerArray::popGet(); }
     inline void replace(TYPE * x, aindex_t pos) { PointerArray::replace(x, pos); }
@@ -204,6 +210,27 @@ public:
     inline bool zap(TYPE * x)                   { return PointerArray::zap(x); }
 };
 
+template <class BTYPE>
+class ConstPointerArrayOf : public ConstPointerArray
+{
+    typedef const BTYPE TYPE;
+    typedef int (*PointerOfCompareFunc)(TYPE **, TYPE **);
+public:
+    inline void add(TYPE * x, aindex_t pos)     { ConstPointerArray::add(x, pos); }
+    inline void append(TYPE * x)                { ConstPointerArray::append(x); }
+    inline aindex_t bAdd(TYPE * & newItem, PointerOfCompareFunc f, bool & isNew) { return ConstPointerArray::bAdd(*(void * *)&newItem, (CompareFunc)f, isNew); }
+    inline aindex_t bSearch(const TYPE * & key, PointerOfCompareFunc f) const    { return ConstPointerArray:: bSearch(*(const void * *)&key, (CompareFunc)f); }
+    inline aindex_t find(TYPE * x) const        { return ConstPointerArray::find(x); }
+    inline TYPE **getArray(aindex_t pos = 0)    { return (TYPE **)ConstPointerArray::getArray(pos); }
+    inline TYPE **detach()                      { return (TYPE **)ConstPointerArray::detach(); }
+    inline TYPE * item(aindex_t pos) const      { return (TYPE *)ConstPointerArray::item(pos); }
+    inline TYPE * popGet()                      { return (TYPE *)ConstPointerArray::popGet(); }
+    inline void replace(TYPE * x, aindex_t pos) { ConstPointerArray::replace(x, pos); }
+    inline TYPE * tos(void) const               { return (TYPE *)ConstPointerArray::tos(); }
+    inline TYPE * tos(aindex_t num) const       { return (TYPE *)ConstPointerArray::tos(num); }
+    inline bool zap(TYPE * x)                   { return ConstPointerArray::zap(x); }
+};
+
 enum DAFSConnectCfg { SSLNone = 0, SSLOnly, SSLFirst, UnsecureFirst };
 
 #include "jstring.hpp"

system/jlib/jptree-attrvalues.hpp

@@ -524,6 +524,7 @@
     "roxie_deploy_map.xml",
     "RoxieServer",
     "roxie.xsd",
+    "_rtlType",
     "run",
     "running",
     "Running",

system/jlib/jptree.ipp

@@ -59,7 +59,7 @@ protected:
         const IPropertyTree &elem = *(const IPropertyTree *)e;
         return elem.queryName();
     }
-    virtual unsigned getHashFromElement(const void *e) const;
+    virtual unsigned getHashFromElement(const void *e) const override;
     virtual unsigned getHashFromFindParam(const void *fp) const override
     {
         return hashc((const unsigned char *)fp, (size32_t)strlen((const char *)fp), 0);
@@ -897,7 +897,7 @@ class CPTreeMaker : public CInterfaceOf<IPTreeMaker>
     {
         byte flags;
     public:
-        IMPLEMENT_IINTERFACE;
+        IMPLEMENT_IINTERFACE_O;
 
         CDefaultNodeCreator(byte _flags) : flags(_flags) { }
 
@@ -986,7 +986,7 @@ public:
         }
         currentNode = NULL;
     }
-    virtual IPropertyTree *create(const char *tag)
+    virtual IPropertyTree *create(const char *tag) override
     {
         return nodeCreator->create(tag);
     }

testing/regress/ecl/badindex.ecl

@@ -0,0 +1,128 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+// Testing various indexes that might present tricky cases for remote projection/filtering
+
+import ^ as root;
+prefix := #IFDEFINED(root.prefix, '~regress::key::badidx');
+
+simple_rec := RECORD
+  unsigned4 u4;
+  integer1 s4;
+  big_endian unsigned4 bu4;
+  big_endian integer4 bs4
+END;
+
+simple_ds := DATASET([{1,1,1,1},{2,2,2,2},{3,-3,3,-3},{4,4,4,4}], simple_rec);
+
+// Simple case - last field gets moved to fileposition and only 3 fields are considered keyed
+simple := INDEX(simple_ds, {simple_ds}, prefix + '1_simple.idx');
+
+// All fields keyed, so no fileposition
+allkeyed := INDEX(simple_ds, {simple_ds}, {}, prefix + '2_allkeyed.idx');
+
+// Only one field, so fileposition
+onekeyed := INDEX(simple_ds, {s4}, prefix + '3_onekeyed.idx');
+ 
+// Check bias - keyed fields get bias (but only if signed), and so do unkeyed (except in filepos field)
+payloadbias := INDEX(simple_ds, {u4},{s4,bu4,bs4}, prefix + '4_payloadbias.idx');
+payloadbias2 := INDEX(simple_ds, {u4},{bu4,bs4,s4}, prefix + '5_payloadbias2.idx');
+
+// Check for tricky types - nested records in payload
+nestrec :=  INDEX(simple_ds, {u4}, {simple_rec r := ROW({u4,s4,bu4,bs4}, simple_rec) }, prefix + '6_nestrec.idx');
+
+// Check for unserializable types in payload
+
+unserialized := INDEX(simple_ds, {u4}, { u4, ifblock(self.u4!=2) s4,END,bu4,bs4 }, prefix + '7_ifrec.idx');
+
+unserialized2 := INDEX(simple_ds, {u4}, { boolean b := u4!=2, ifblock(self.b) s4,bu4,bs4 END }, prefix + '8_ifrec.idx');
+
+// Check for nested ifblocks
+
+unserialized3 := INDEX(simple_ds, {u4}, { u4, ifblock(self.u4!=2) s4, ifblock(self.u4 != 2) bu4 END,END,bs4 }, prefix + '9_ifrec.idx');
+
+// Check for tricky types - child datasets 
+// Check for tricky types - utf8 
+// Check for tricky types - unicode 
+// Check for tricky types - blobs
+// Check for tricky types - alien data types 
+// Check for tricky types - sets in payload 
+
+set_rec := RECORD
+  unsigned4 u4;
+  set of integer1 s4;
+  set of big_endian unsigned4 bu4;
+  set of big_endian integer4 bs4
+END;
+
+set_ds := DATASET([{1,[1,2,3],[1,2,3],[1,2,3]},{2,[2,3,4],[2,3,4],[2,3,4]},{3,[-3,-4,-5],[3,4,5],[-3,-4,-5]},{4,[4,5,6],[4,5,6],[4,5,6]}], set_rec);
+
+keyedset := INDEX(set_ds, {u4},{s4,bu4,bs4 }, prefix + '10_setrec.idx');
+
+// Check for multipart keys, noroot keys, etc
+
+SEQUENTIAL(
+  OUTPUT('simple'),
+  BUILDINDEX(simple, OVERWRITE),
+  OUTPUT(choosen(simple, 5));
+  //OUTPUT(choosen(simple(keyed(bu4=3),WILD(u4),WILD(s4)), 5)), 
+
+  OUTPUT('allkeyed'),
+  BUILDINDEX(allkeyed, OVERWRITE),
+  OUTPUT(choosen(allkeyed, 5));
+  //OUTPUT(choosen(allkeyed(keyed(bs4=-3),WILD(u4),WILD(s4),WILD(bu4)), 5)), 
+
+  OUTPUT('onekeyed'),
+  BUILDINDEX(onekeyed, OVERWRITE),
+  OUTPUT(choosen(onekeyed, 5)); 
+  //OUTPUT(choosen(onekeyed(keyed(s4=-3)), 5)),
+
+  OUTPUT('payloadbias'),
+  BUILDINDEX(payloadbias, OVERWRITE),
+  OUTPUT(choosen(payloadbias, 5)); 
+  //OUTPUT(choosen(payloadbias(keyed(u4=3)), 5)) 
+
+  OUTPUT('payloadbias2'),
+  BUILDINDEX(payloadbias2, OVERWRITE),
+  OUTPUT(choosen(payloadbias2, 5)); 
+  //OUTPUT(choosen(payloadbias2(keyed(u4=3)), 5)) 
+
+  OUTPUT('nestrec'),
+  BUILDINDEX(nestrec, OVERWRITE),
+  OUTPUT(choosen(nestrec, 5)); 
+  //OUTPUT(choosen(nestrec(keyed(u4=3)), 5)) 
+
+  OUTPUT('unserialized'),
+  BUILDINDEX(unserialized, OVERWRITE),
+  OUTPUT(choosen(unserialized, 5)); 
+  //OUTPUT(choosen(unserialized(keyed(u4=3)), 5)) 
+
+  OUTPUT('unserialized2'),
+  BUILDINDEX(unserialized2, OVERWRITE),
+  OUTPUT(choosen(unserialized2, 5)); 
+  //OUTPUT(choosen(unserialized2(keyed(u4=3)), 5)) 
+
+  OUTPUT('unserialized3'),
+  BUILDINDEX(unserialized3, OVERWRITE),
+  OUTPUT(choosen(unserialized3, 5)); 
+  //OUTPUT(choosen(unserialized3(keyed(u4=3)), 5)) 
+
+  OUTPUT('keyedset'),
+  BUILDINDEX(keyedset, OVERWRITE),
+  OUTPUT(choosen(keyedset, 5)); 
+  //OUTPUT(choosen(keyedset(keyed(u4=[3,4,5])), 5)) 
+);

testing/regress/ecl/key/badindex.xml

@@ -0,0 +1,110 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>simple</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><u4>1</u4><s4>1</s4><bu4>1</bu4><bs4>1</bs4></Row>
+ <Row><u4>2</u4><s4>2</s4><bu4>2</bu4><bs4>2</bs4></Row>
+ <Row><u4>3</u4><s4>-3</s4><bu4>3</bu4><bs4>-3</bs4></Row>
+ <Row><u4>4</u4><s4>4</s4><bu4>4</bu4><bs4>4</bs4></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>allkeyed</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><u4>1</u4><s4>1</s4><bu4>1</bu4><bs4>1</bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>2</u4><s4>2</s4><bu4>2</bu4><bs4>2</bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>3</u4><s4>-3</s4><bu4>3</bu4><bs4>-3</bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>4</u4><s4>4</s4><bu4>4</bu4><bs4>4</bs4><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>onekeyed</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><s4>-3</s4></Row>
+ <Row><s4>1</s4></Row>
+ <Row><s4>2</s4></Row>
+ <Row><s4>4</s4></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>payloadbias</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><u4>1</u4><s4>1</s4><bu4>1</bu4><bs4>1</bs4></Row>
+ <Row><u4>2</u4><s4>2</s4><bu4>2</bu4><bs4>2</bs4></Row>
+ <Row><u4>3</u4><s4>-3</s4><bu4>3</bu4><bs4>-3</bs4></Row>
+ <Row><u4>4</u4><s4>4</s4><bu4>4</bu4><bs4>4</bs4></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>payloadbias2</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><u4>1</u4><bu4>1</bu4><bs4>1</bs4><s4>1</s4></Row>
+ <Row><u4>2</u4><bu4>2</bu4><bs4>2</bs4><s4>2</s4></Row>
+ <Row><u4>3</u4><bu4>3</bu4><bs4>-3</bs4><s4>-3</s4></Row>
+ <Row><u4>4</u4><bu4>4</bu4><bs4>4</bs4><s4>4</s4></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>nestrec</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><u4>1</u4><r><u4>1</u4><s4>1</s4><bu4>1</bu4><bs4>1</bs4></r><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>2</u4><r><u4>2</u4><s4>2</s4><bu4>2</bu4><bs4>2</bs4></r><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>3</u4><r><u4>3</u4><s4>-3</s4><bu4>3</bu4><bs4>-3</bs4></r><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>4</u4><r><u4>4</u4><s4>4</s4><bu4>4</bu4><bs4>4</bs4></r><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>unserialized</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><u4>1</u4><s4>1</s4><bu4>1</bu4><bs4>1</bs4></Row>
+ <Row><u4>2</u4><bu4>2</bu4><bs4>2</bs4></Row>
+ <Row><u4>3</u4><s4>-3</s4><bu4>3</bu4><bs4>-3</bs4></Row>
+ <Row><u4>4</u4><s4>4</s4><bu4>4</bu4><bs4>4</bs4></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>unserialized2</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><u4>1</u4><b>true</b><s4>1</s4><bu4>1</bu4><bs4>1</bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>2</u4><b>false</b><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>3</u4><b>true</b><s4>-3</s4><bu4>3</bu4><bs4>-3</bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>4</u4><b>true</b><s4>4</s4><bu4>4</bu4><bs4>4</bs4><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><Result_25>unserialized3</Result_25></Row>
+</Dataset>
+<Dataset name='Result 26'>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><u4>1</u4><s4>1</s4><bu4>1</bu4><bs4>1</bs4></Row>
+ <Row><u4>2</u4><bs4>2</bs4></Row>
+ <Row><u4>3</u4><s4>-3</s4><bu4>3</bu4><bs4>-3</bs4></Row>
+ <Row><u4>4</u4><s4>4</s4><bu4>4</bu4><bs4>4</bs4></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><Result_28>keyedset</Result_28></Row>
+</Dataset>
+<Dataset name='Result 29'>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><u4>1</u4><s4><Item>1</Item><Item>2</Item><Item>3</Item></s4><bu4><Item>1</Item><Item>2</Item><Item>3</Item></bu4><bs4><Item>1</Item><Item>2</Item><Item>3</Item></bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>2</u4><s4><Item>2</Item><Item>3</Item><Item>4</Item></s4><bu4><Item>2</Item><Item>3</Item><Item>4</Item></bu4><bs4><Item>2</Item><Item>3</Item><Item>4</Item></bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>3</u4><s4><Item>-3</Item><Item>-4</Item><Item>-5</Item></s4><bu4><Item>3</Item><Item>4</Item><Item>5</Item></bu4><bs4><Item>-3</Item><Item>-4</Item><Item>-5</Item></bs4><__internal_fpos__>0</__internal_fpos__></Row>
+ <Row><u4>4</u4><s4><Item>4</Item><Item>5</Item><Item>6</Item></s4><bu4><Item>4</Item><Item>5</Item><Item>6</Item></bu4><bs4><Item>4</Item><Item>5</Item><Item>6</Item></bs4><__internal_fpos__>0</__internal_fpos__></Row>
+</Dataset>

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -25,6 +25,7 @@
 #include "ctfile.hpp"
 #include "eclrtl.hpp"
 #include "thorfile.hpp"
+#include "rtldynfield.hpp"
 
 class IndexWriteActivityMaster : public CMasterActivity
 {
@@ -168,6 +169,7 @@ public:
         const char *rececl= helper->queryRecordECL();
         if (rececl&&*rececl)
             props.setProp("ECL", rececl);
+        // Legacy record layout info
         void * layoutMetaBuff;
         size32_t layoutMetaSize;
         if(helper->getIndexLayout(layoutMetaSize, layoutMetaBuff))
@@ -175,6 +177,13 @@ public:
             props.setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
             rtlFree(layoutMetaBuff);
         }
+        // New record layout info
+        if (helper->queryOutputMeta() && helper->queryOutputMeta()->queryTypeInfo())
+        {
+            MemoryBuffer out;
+            dumpTypeInfo(out, helper->queryOutputMeta()->queryTypeInfo(), true);
+            props.setPropBin("_rtlType", out.length(), out.toByteArray());
+        }
         mpTag = container.queryJob().allocateMPTag();
         mpTag2 = container.queryJob().allocateMPTag();
     }

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -28,6 +28,7 @@
 #include "keybuild.hpp"
 #include "thbufdef.hpp"
 #include "backup.hpp"
+#include "rtldynfield.hpp"
 
 #define SINGLEPART_KEY_TRANSFER_SIZE 0x10000
 #define FEWWARNCAP 10
@@ -201,16 +202,15 @@ public:
         if(!metadata) metadata.setown(createPTree("metadata"));
         metadata->setProp("_record_ECL", helper->queryRecordECL());
 
-        void * layoutMetaBuff;
-        size32_t layoutMetaSize;
-        if(helper->getIndexLayout(layoutMetaSize, layoutMetaBuff))
+        if (helper->queryOutputMeta() && helper->queryOutputMeta()->queryTypeInfo())
         {
-            metadata->setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
-            rtlFree(layoutMetaBuff);
+            MemoryBuffer out;
+            dumpTypeInfo(out, helper->queryOutputMeta()->queryTypeInfo(), true);
+            metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }
 
-    void close(IPartDescriptor &partDesc, unsigned &crc, bool addMeta=false)
+    void close(IPartDescriptor &partDesc, unsigned &crc)
     {
         StringBuffer partFname;
         getPartFilename(partDesc, 0, partFname);
@@ -218,14 +218,7 @@ public:
         try
         {
             if (builder)
-            {
-                if (addMeta && metadata)
-                {
-                    builder->finish(metadata, &crc);
-                }
-                else
-                    builder->finish(&crc);
-            }
+                builder->finish(metadata, &crc);
         }
         catch (IException *_e)
         {
@@ -365,10 +358,10 @@ public:
                 }
                 catch (CATCHALL)
                 {
-                    close(*partDesc, partCrc, true);
+                    close(*partDesc, partCrc);
                     throw;
                 }
-                close(*partDesc, partCrc, true);
+                close(*partDesc, partCrc);
                 stop();
             }
             else
@@ -423,10 +416,10 @@ public:
                 }
                 catch (CATCHALL)
                 {
-                    close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
+                    close(*partDesc, partCrc);
                     throw;
                 }
-                close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
+                close(*partDesc, partCrc);
                 stop();
 
                 ActPrintLog("INDEXWRITE: Wrote %" RCPF "d records", processed & THORDATALINK_COUNT_MASK);
@@ -494,12 +487,12 @@ public:
                                 CNodeInfo &info = tlkRows.item(idx);
                                 builder->processKeyData((char *)info.value, info.pos, info.size);
                             }
-                            close(*tlkDesc, tlkCrc, true);
+                            close(*tlkDesc, tlkCrc);
                         }
                         catch (CATCHALL)
                         {
                             abortSoon = true;
-                            close(*tlkDesc, tlkCrc, true);
+                            close(*tlkDesc, tlkCrc);
                             removeFiles(*partDesc);
                             throw;
                         }

thorlcr/activities/keypatch/thkeypatch.cpp

@@ -161,8 +161,12 @@ public:
         if (originalProps.queryProp("ECL"))
             props.setProp("ECL", originalProps.queryProp("ECL"));
         MemoryBuffer rLMB;
+        // Legacy record layout info
         if (originalProps.getPropBin("_record_layout", rLMB))
             props.setPropBin("_record_layout", rLMB.length(), rLMB.toByteArray());
+        // New record layout info
+        if (originalProps.getPropBin("_rtlType", rLMB.clear()))
+            props.setPropBin("_rtlType", rLMB.length(), rLMB.toByteArray());
         props.setPropInt("@formatCrc", originalProps.getPropInt("@formatCrc"));
         if (originalProps.getPropBool("@local"))
             props.setPropBool("@local", true);

tools/dumpkey/CMakeLists.txt

@@ -31,8 +31,10 @@ set (    SRCS
 
 include_directories ( 
          ./../../system/jhtree 
-         ./../../rtl/eclrtl 
+         ./../../rtl/include
+         ./../../rtl/eclrtl
          ./../../system/include 
+         ./../../common/thorhelper 
          ./../../system/jlib 
     )
 
@@ -41,7 +43,9 @@ ADD_DEFINITIONS( -D_CONSOLE )
 HPCC_ADD_EXECUTABLE ( dumpkey ${SRCS} )
 install ( TARGETS dumpkey RUNTIME DESTINATION ${EXEC_DIR} )
 
-target_link_libraries ( dumpkey 
+target_link_libraries ( dumpkey
+         eclrtl
+         thorhelper
          jlib
          jhtree
     )

tools/dumpkey/dumpkey.cpp

@@ -18,6 +18,9 @@
 #include "jliball.hpp"
 #include "jhtree.hpp"
 #include "ctfile.hpp"
+#include "rtlrecord.hpp"
+#include "rtlformat.hpp"
+#include "eclhelper_dyn.hpp"
 
 void fatal(const char *format, ...) __attribute__((format(printf, 1, 2)));
 void fatal(const char *format, ...)
@@ -35,16 +38,23 @@ void fatal(const char *format, ...)
 
 bool optHex = false;
 bool optRaw = false;
+bool optFullHeader = false;
+bool optHeader = false;
+StringArray files;
 
 void usage()
 {
-    fprintf(stderr, "Usage: dumpkey dataset [options]\n"
+    fprintf(stderr, "Usage: dumpkey [options] dataset [dataset...]\n"
         "Options:\n"
         "  node=[n]            - dump node n (0 = just header)\n"
         "  fpos=[n]            - dump node at offset fpos\n"
-        "  recs=[n]            - dump n rows\n"
+        "  recs=[n]            - output n rows\n"
+        "  fields=[fieldnames] - output specified fields only\n"
+        "  filter=[filter]     - filter rows\n"
         "  -H                  - hex display\n"
         "  -R                  - raw output\n"
+        "  -fullheader         - output full header info for each file\n"
+        "  -header             - output minimal header info for each file\n"
                     );
     fflush(stderr);
     releaseAtoms();
@@ -54,17 +64,16 @@ void usage()
 
 void doOption(const char *opt)
 {
-    switch (toupper(opt[1]))
-    {
-    case 'H':
+    if (streq(opt, "-H"))
         optHex = true;
-        break;
-    case 'R':
+    else if (streq(opt, "-R"))
         optRaw = true;
-        break;
-    default:
+    else if (streq(opt, "-header"))
+        optHeader = true;
+    else if (streq(opt, "-fullheader"))
+        optFullHeader = true;
+    else
         usage();
-    }
 }
 
 int main(int argc, const char **argv)
@@ -75,116 +84,187 @@ int main(int argc, const char **argv)
     _setmode( _fileno( stdin ), _O_BINARY );
 #endif
     Owned<IProperties> globals = createProperties("dumpkey.ini", true);
+    StringArray filters;
     for (int i = 1; i < argc; i++)
     {
         if (argv[i][0] == '-')
             doOption(argv[i]);
+        else if (strncmp(argv[i], "filter=", 7)==0)
+            filters.append(argv[i]+7);
         else if (strchr(argv[i], '='))
             globals->loadProp(argv[i]);
         else
-            globals->setProp("keyfile", argv[i]);
+            files.append(argv[i]);
     }
-    try
+    StringBuffer logname("dumpkey.");
+    logname.append(GetCachedHostName()).append(".");
+    StringBuffer lf;
+    openLogFile(lf, logname.append("log").str());
+    ForEachItemIn(idx, files)
     {
-        StringBuffer logname("dumpkey.");
-        logname.append(GetCachedHostName()).append(".");
-        StringBuffer lf;
-        openLogFile(lf, logname.append("log").str());
-        
-        Owned <IKeyIndex> index;
-        const char * keyName = globals->queryProp("keyfile");
-        if (keyName)
-            index.setown(createKeyIndex(keyName, 0, false, false));
-        else
-            usage();
-        Owned <IKeyCursor> cursor = index->getCursor(NULL);
-
-        size32_t key_size = index->keySize();
-        Owned<IFile> in = createIFile(keyName);
-        Owned<IFileIO> io = in->open(IFOread);
-        if (!io)
-            throw MakeStringException(999, "Failed to open file %s", keyName);
-        Owned<CKeyHdr> header = new CKeyHdr;
-        MemoryAttr block(sizeof(KeyHdr));
-        io->read(0, sizeof(KeyHdr), (void *)block.get());
-        header->load(*(KeyHdr*)block.get());
-        unsigned nodeSize = header->getNodeSize();
-
-        if (!optRaw)
-        {
-            printf("Key '%s'\nkeySize=%d NumParts=%x, Top=%d\n", keyName, key_size, index->numParts(), index->isTopLevelKey());
-            printf("File size = %" I64F "d, nodes = %" I64F "d\n", in->size(), in->size() / nodeSize - 1);
-            printf("rootoffset=%" I64F "d[%" I64F "d]\n", header->getRootFPos(), header->getRootFPos()/nodeSize);
-        }
-        char *buffer = (char*)alloca(key_size);
-
-        if (globals->hasProp("node"))
+        try
         {
-            if (stricmp(globals->queryProp("node"), "all")==0)
+            Owned <IKeyIndex> index;
+            const char * keyName = files.item(idx);
+            index.setown(createKeyIndex(keyName, 0, false, false));
+            size32_t key_size = index->keySize();  // NOTE - in variable size case, this is 32767
+            unsigned nodeSize = index->getNodeSize();
+            if (optFullHeader)
             {
+                Owned<IFile> in = createIFile(keyName);
+                Owned<IFileIO> io = in->open(IFOread);
+                if (!io)
+                    throw MakeStringException(999, "Failed to open file %s", keyName);
+                Owned<CKeyHdr> header = new CKeyHdr;
+                MemoryAttr block(sizeof(KeyHdr));
+                io->read(0, sizeof(KeyHdr), (void *)block.get());
+                header->load(*(KeyHdr*)block.get());
+
+                printf("Key '%s'\nkeySize=%d NumParts=%x, Top=%d\n", keyName, key_size, index->numParts(), index->isTopLevelKey());
+                printf("File size = %" I64F "d, nodes = %" I64F "d\n", in->size(), in->size() / nodeSize - 1);
+                printf("rootoffset=%" I64F "d[%" I64F "d]\n", header->getRootFPos(), header->getRootFPos()/nodeSize);
+                Owned<IPropertyTree> metadata = index->getMetadata();
+                if (metadata)
+                {
+                    StringBuffer xml;
+                    toXML(metadata, xml);
+                    printf("MetaData:\n%s\n", xml.str());
+                }
             }
-            else
+            else if (optHeader)
             {
-                int node = globals->getPropInt("node");
-                if (node != 0)
-                    index->dumpNode(stdout, node * nodeSize, globals->getPropInt("recs", 0), optRaw);
+                if (idx)
+                    printf("\n");
+                printf("%s:\n\n", keyName);
             }
-        }
-        else if (globals->hasProp("fpos"))
-        {
-            index->dumpNode(stdout, globals->getPropInt("fpos"), globals->getPropInt("recs", 0), optRaw);
-        }
-        else
-        {
-            bool backwards=false;
-            bool ok;
-            if (globals->hasProp("end"))
+
+            if (globals->hasProp("node"))
             {
-                memset(buffer, 0, key_size);
-                strcpy(buffer, globals->queryProp("end"));
-                ok = cursor->ltEqual(buffer, buffer);
-                backwards = true;
+                if (stricmp(globals->queryProp("node"), "all")==0)
+                {
+                }
+                else
+                {
+                    int node = globals->getPropInt("node");
+                    if (node != 0)
+                        index->dumpNode(stdout, node * nodeSize, globals->getPropInt("recs", 0), optRaw);
+                }
             }
-            else if (globals->hasProp("start"))
+            else if (globals->hasProp("fpos"))
             {
-                memset(buffer, 0, key_size);
-                strcpy(buffer, globals->queryProp("start"));
-                ok = cursor->gtEqual(buffer, buffer);
+                index->dumpNode(stdout, globals->getPropInt("fpos"), globals->getPropInt("recs", 0), optRaw);
             }
             else
-                ok = cursor->first(buffer);
-            
-            unsigned count = globals->getPropInt("recs", 1);
-            while (ok && count--)
             {
-                offset_t pos = cursor->getFPos();
-                unsigned __int64 seq = cursor->getSequence();
-                size32_t size = cursor->getSize();
-                if (optRaw)
+                Owned<IKeyManager> manager = createLocalKeyManager(index, key_size, NULL);
+                Owned<IPropertyTree> metadata = index->getMetadata();
+                Owned<IOutputMetaData> diskmeta;
+                Owned<IOutputMetaData> translatedmeta;
+                ArrayOf<const RtlFieldInfo *> fields;  // Note - the lifetime of the array needs to extend beyond the lifetime of outmeta. The fields themselves are shared with diskmeta, and do not need to be released.
+                Owned<IOutputMetaData> outmeta;
+                Owned<IHThorIndexReadArg> helper;
+                Owned<IXmlWriterExt> writer;
+                class MyIndexCallback : public CInterfaceOf<IThorIndexCallback>
+                {
+                public:
+                    MyIndexCallback(IKeyManager *_manager) : manager(_manager) {}
+                    virtual unsigned __int64 getFilePosition(const void * row)
+                    {
+                        return manager->queryFpos();
+                    }
+                    virtual byte * lookupBlob(unsigned __int64 id)
+                    {
+                        UNIMPLEMENTED;
+                    }
+                    Linked<IKeyManager> manager;
+                } callback(manager);
+                unsigned count = globals->getPropInt("recs", 1);
+                const RtlRecordTypeInfo *outRecType = nullptr;
+                if (metadata && metadata->hasProp("_rtlType"))
                 {
-                    fwrite(buffer, 1, size, stdout);
+                    MemoryBuffer layoutBin;
+                    metadata->getPropBin("_rtlType", layoutBin);
+                    diskmeta.setown(createTypeInfoOutputMetaData(layoutBin, &callback));
+                    writer.setown(new SimpleOutputWriter);
+                    if (globals->hasProp("fields"))
+                    {
+                        StringArray fieldNames;
+                        fieldNames.appendList(globals->queryProp("fields"), ",");
+                        const RtlRecord &inrec = diskmeta->queryRecordAccessor(true);
+                        size32_t minRecSize = 0;
+                        ForEachItemIn(idx, fieldNames)
+                        {
+                            unsigned fieldNum = inrec.getFieldNum(fieldNames.item(idx));
+                            if (fieldNum == (unsigned) -1)
+                                throw MakeStringException(0, "Requested output field '%s' not found", fieldNames.item(idx));
+                            fields.append(inrec.queryOriginalField(fieldNum));
+                            minRecSize += inrec.queryType(fieldNum)->getMinSize();
+                        }
+                        fields.append(nullptr);
+                        outRecType = new RtlRecordTypeInfo(type_record, minRecSize, fields.getArray(0));
+                        outmeta.setown(new CDynamicOutputMetaData(*outRecType));
+                    }
+                    else
+                        outmeta.set(diskmeta);
+                    helper.setown(createIndexReadArg(keyName, diskmeta.getLink(), outmeta.getLink(), count, 0, (uint64_t) -1));
+                    helper->setCallback(&callback);
+                    if (filters.ordinality())
+                    {
+                        IDynamicIndexReadArg *arg = QUERYINTERFACE(helper.get(), IDynamicIndexReadArg);
+                        assertex(arg);
+                        ForEachItemIn(idx, filters)
+                        {
+                            arg->addFilter(filters.item(idx));
+                        }
+                    }
+                    helper->createSegmentMonitors(manager);
+                    count = helper->getChooseNLimit(); // Just because this is testing out the createIndexReadArg functionality
                 }
-                else if (optHex)
+                manager->finishSegmentMonitors();
+                manager->reset();
+                while (manager->lookup(true) && count--)
                 {
-                    for (unsigned i = 0; i < size; i++)
-                        printf("%02x", ((unsigned char) buffer[i]) & 0xff);
-                    printf("  :%" I64F "u:%012" I64F "x\n", seq, pos);
+                    offset_t pos;
+                    byte const * buffer = manager->queryKeyBuffer(pos);
+                    size32_t size = manager->queryRowSize();
+                    unsigned __int64 seq = manager->querySequence();
+                    if (optRaw)
+                    {
+                        fwrite(buffer, 1, size, stdout);
+                    }
+                    else if (optHex)
+                    {
+                        for (unsigned i = 0; i < size; i++)
+                            printf("%02x", ((unsigned char) buffer[i]) & 0xff);
+                        printf("  :%" I64F "u:%012" I64F "x\n", seq, pos);
+                    }
+                    else if (helper)
+                    {
+                        MemoryBuffer buf;
+                        MemoryBufferBuilder aBuilder(buf, 0);
+                        if (helper->transform(aBuilder, (const byte *) buffer))
+                        {
+                            outmeta->toXML((const byte *) buf.toByteArray(), *writer.get());
+                            printf("%s\n", writer->str());
+                            writer->clear();
+                        }
+                        else
+                            count++;  // Don't count this row as it was postfiltered
+                    }
+                    else
+                        printf("%.*s  :%" I64F "u:%012" I64F "x\n", size, buffer, seq, pos);
                 }
-                else
-                    printf("%.*s  :%" I64F "u:%012" I64F "x\n", size, buffer, seq, pos);
-                if (backwards)
-                    ok = cursor->prev(buffer);
-                else
-                    ok = cursor->next(buffer);
+                if (outRecType)
+                    outRecType->doDelete();
             }
         }
-    }
-    catch (IException *E)
-    {
-        StringBuffer msg;
-        E->errorMessage(msg);
-        E->Release();
-        fatal("%s", msg.str());
+        catch (IException *E)
+        {
+            StringBuffer msg;
+            E->errorMessage(msg);
+            printf("%s\n", msg.str());
+            E->Release();
+        }
     }
     releaseAtoms();
     ExitModuleObjects();