HPCC-19999 Record translator should support de-blobbing BLOB fields

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -134,87 +134,6 @@ unsigned promotedIntSize[9] = { 0, 1, 2, 4, 4, 8, 8, 8, 8 };
 
 //===========================================================================
 
-
-ITypeInfo *makeType(type_t type, int size)
-{
-    switch (type)
-    {
-    case type_string:
-        return makeStringType(size, NULL, NULL);            // MORE!!
-        
-    case type_varstring:
-        return makeVarStringType(size);
-        
-    case type_qstring:
-        return makeQStringType(size);
-        
-    case type_unicode:
-        return makeUnicodeType(size, 0);
-
-    case type_varunicode:
-        return makeVarUnicodeType(size, 0);
-
-    case type_utf8:
-        return makeUtf8Type(size, 0);
-
-    case type_data:
-        return makeDataType(size);
-        
-    case type_int:
-        return makeIntType(size, false);
-        
-    case type_swapint:
-        return makeSwapIntType(size, false);
-        
-    case type_packedint:
-        return makePackedIntType(size, false);
-        
-    case type_real:
-        return makeRealType(size);
-        
-    case type_bitfield:
-        return makeBitfieldType(size);
-        
-    case type_boolean:
-        return makeBoolType();
-
-    case type_blob:
-        return makeBlobType();
-
-    case type_void:
-        return makeVoidType();
-        
-    case type_null:
-        return makeNullType();
-        
-    case type_record:
-        return makeRecordType();
-        
-    case type_pattern:
-        return makePatternType();
-
-    case type_rule:
-        return makeRuleType(NULL);
-
-    case type_token:
-        return makeTokenType();
-
-    case type_feature:
-        return makeFeatureType();
-
-    case type_event:
-        return makeEventType();
-
-    case type_any:
-        return makeAnyType();
-        
-    case type_date:
-        return makeBoolType(); // JCSMORE: Todo.
-    }
-    return NULL;
-}
-
-
 static IValue * castViaString(ITypeInfo * type, size32_t len, const char * text)
 {
     Owned<IValue> temp = createStringValue(text, len);
@@ -1905,6 +1824,11 @@ static ITypeInfo * commonUpType(CHashedTypeInfo * candidate)
     return match;
 }
 
+extern DEFTYPE_API ITypeInfo *makeKeyedBlobType(ITypeInfo * basetype)
+{
+    return commonUpType(new CKeyedBlobTypeInfo(basetype));
+}
+
 extern DEFTYPE_API ITypeInfo *makeFilePosType(ITypeInfo *basetype)
 {
     assertex(basetype);

common/deftype/deftype.hpp

@@ -177,12 +177,12 @@ extern DEFTYPE_API ITypeInfo *makeDataType(int size);
 extern DEFTYPE_API ITypeInfo *makeBitfieldType(int sizeInBits, ITypeInfo * basetype = NULL);
 extern DEFTYPE_API ITypeInfo *makeBoolType();
 extern DEFTYPE_API ITypeInfo *makeBlobType();
+extern DEFTYPE_API ITypeInfo *makeKeyedBlobType(ITypeInfo * basetype);
 extern DEFTYPE_API ITypeInfo *makeRecordType();     // not used by any IHqlExpressions
 extern DEFTYPE_API ITypeInfo *makeVoidType();
 extern DEFTYPE_API ITypeInfo *makeNullType();
 extern DEFTYPE_API ITypeInfo *makeEventType();
 extern DEFTYPE_API ITypeInfo *makeAnyType();
-extern DEFTYPE_API ITypeInfo *makeType(type_t type, int size);
 extern DEFTYPE_API IEnumeratedTypeBuilder *makeEnumeratedTypeBuilder(ITypeInfo *base, aindex_t numvalues);
 extern DEFTYPE_API ITypeInfo *makeDecimalType(unsigned digits, unsigned prec, bool isSigned);
 extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype);

common/deftype/deftype.ipp

@@ -559,6 +559,28 @@ protected:
     Owned<ITypeInfo>    basetype;
 };
 
+class CKeyedBlobTypeInfo : public CBasedTypeInfo
+{
+public:
+
+    virtual void serialize(MemoryBuffer &tgt) { CTypeInfo::serialize(tgt); }
+public:
+    CKeyedBlobTypeInfo(ITypeInfo * _basetype) : CBasedTypeInfo(_basetype, sizeof(offset_t)) {}
+    virtual type_t getTypeCode() const { return type_blob; };
+
+    // Only used for generation of type information so no need to fully implement these
+    virtual bool isSwappedEndian()              { return false; }
+    virtual bool isInteger()                    { return true; };
+    virtual bool isScalar()                     { return true; }
+    virtual bool isSigned()                     { return false; }
+    virtual unsigned getStringLen()             { return basetype->getStringLen(); }
+    virtual unsigned getDigits()                { return 0; }
+    virtual const char *queryTypeName()         { return "blob"; }
+    virtual ITypeInfo * queryPromotedType()     { return basetype->queryPromotedType(); }
+    virtual StringBuffer &getECLType(StringBuffer & out) { return out.append("blob"); }
+    virtual unsigned getCrc()                   { return basetype->getCrc(); }
+};
+
 class CFilePosTypeInfo : public CBasedTypeInfo
 {
 public:

common/fileview2/fvidxsource.cpp

@@ -124,7 +124,7 @@ IndexDataSource::IndexDataSource(const char * _logicalName, IHqlExpression * _di
 {
     logicalName.set(_logicalName);
     diskRecord.set(_diskRecord);
-    deserializer.setown(createRtlFieldTypeDeserializer(nullptr));
+    deserializer.setown(createRtlFieldTypeDeserializer());
     diskRecordMeta.setown(new CDynamicOutputMetaData(* static_cast<const RtlRecordTypeInfo *>(queryRtlType(*deserializer.get(), diskRecord))));
     Owned<IUserDescriptor> udesc;
     if(_username != NULL && *_username != '\0')

common/remote/sockfile.cpp

@@ -3940,7 +3940,7 @@ static IOutputMetaData *getTypeInfoOutputMetaData(IPropertyTree &actNode, const
 {
     IPropertyTree *json = actNode.queryPropTree(typePropName);
     if (json)
-        return createTypeInfoOutputMetaData(*json, grouped, nullptr);
+        return createTypeInfoOutputMetaData(*json, grouped);
     else
     {
         StringBuffer binTypePropName(typePropName);
@@ -3949,7 +3949,7 @@ static IOutputMetaData *getTypeInfoOutputMetaData(IPropertyTree &actNode, const
             return nullptr;
         MemoryBuffer mb;
         JBASE64_Decode(jsonBin, mb);
-        return createTypeInfoOutputMetaData(mb, grouped, nullptr);
+        return createTypeInfoOutputMetaData(mb, grouped);
     }
 }
 
@@ -4021,6 +4021,10 @@ public:
     {
         throwUnexpected();
     }
+    virtual byte * lookupBlob(unsigned __int64 id) override
+    {
+        throwUnexpected();
+    }
 };
 
 class CRemoteDiskReadActivity : public CRemoteDiskBaseActivity
@@ -4214,6 +4218,10 @@ public:
     {
         return makeLocalFposOffset(partNum, prefetchBuffer.tell());
     }
+    virtual byte * lookupBlob(unsigned __int64 id) override
+    {
+        throwUnexpected();
+    }
 };
 
 

common/thorhelper/thorcommon.cpp

@@ -2001,14 +2001,23 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
 {
     try
     {
+        Owned<IException> error;
         bool isGrouped = props.getPropBool("@grouped", false);
         if (props.hasProp("_rtlType"))
         {
             MemoryBuffer layoutBin;
             props.getPropBin("_rtlType", layoutBin);
-            return createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr);
+            try
+            {
+                return createTypeInfoOutputMetaData(layoutBin, isGrouped);
+            }
+            catch (IException *E)
+            {
+                EXCLOG(E);
+                error.setown(E); // Save to throw later if we can't recover via ECL
+            }
         }
-        else if (props.hasProp("ECL"))
+        if (props.hasProp("ECL"))
         {
             const char *kind = props.queryProp("@kind");
             bool isIndex = (kind && streq(kind, "key"));
@@ -2026,9 +2035,13 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
                 }
                 MemoryBuffer layoutBin;
                 if (exportBinaryType(layoutBin, expr, isIndex))
-                    return createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr);
+                    return createTypeInfoOutputMetaData(layoutBin, isGrouped);
             }
         }
+        if (error)
+        {
+            throw(error.getClear());
+        }
     }
     catch (IException *E)
     {

common/thorhelper/thorcommon.ipp

@@ -721,6 +721,7 @@ class NullDiskCallback : public IThorDiskCallback, extends CInterface
     virtual unsigned __int64 getFilePosition(const void * row) { return 0; }
     virtual unsigned __int64 getLocalFilePosition(const void * row) { return 0; }
     virtual const char * queryLogicalFilename(const void * row) { return NULL; }
+    virtual const byte * lookupBlob(unsigned __int64 id) { return nullptr; }
 };
 
 extern THORHELPER_API size32_t cloneRow(ARowBuilder & rowBuilder, const void * row, IOutputMetaData * meta);

common/thorhelper/thorpipe.cpp

@@ -255,6 +255,8 @@ public:
     virtual unsigned __int64 getFilePosition(const void * row) { return 0; }
     virtual unsigned __int64 getLocalFilePosition(const void * row) { return 0; }
     virtual const char * queryLogicalFilename(const void * row) { return ""; }
+    virtual const byte * lookupBlob(unsigned __int64 id) override { throwUnexpected(); }
+
 
 private:
     IXmlToRowTransformer * xmlTransformer;

ecl/hql/hqlexpr.cpp

@@ -14353,7 +14353,7 @@ void exportJsonType(StringBuffer &ret, IHqlExpression *table, bool forceIndex)
     }
     else
     {
-        Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+        Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
         const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
         dumpTypeInfo(ret, typeInfo);
     }
@@ -14372,7 +14372,7 @@ bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table, bool forceIndex)
     {
         try
         {
-            Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+            Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
             const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
             return dumpTypeInfo(ret, typeInfo);
         }

ecl/hql/hqlstack.cpp

@@ -249,7 +249,7 @@ int FuncCallStack::push(ITypeInfo* argType, IHqlExpression* curParam)
 int FuncCallStack::pushMeta(ITypeInfo *type)
 {
     if (!deserializer)
-        deserializer.setown(createRtlFieldTypeDeserializer(nullptr));
+        deserializer.setown(createRtlFieldTypeDeserializer());
     const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), type);
     CDynamicOutputMetaData * meta = new CDynamicOutputMetaData(* static_cast<const RtlRecordTypeInfo *>(typeInfo));
     metas.append(*meta);

ecl/hql/hqlutil.cpp

@@ -424,7 +424,10 @@ IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpre
             physicalFields.append(*createPhysicalIndexRecord(mapper, cur, false, createKeyedTypes));
         else if (cur->hasAttribute(blobAtom))
         {
-            newField = createField(cur->queryId(), makeIntType(8, false), nullptr, createKeyedTypes ? createAttribute(_payload_Atom) : nullptr);
+            if (createKeyedTypes)
+                newField = createField(cur->queryId(), makeKeyedBlobType(cur->getType()), nullptr, createAttribute(_payload_Atom));
+            else
+                newField = createField(cur->queryId(), makeIntType(8, false), nullptr, nullptr);
         }
         else
         {

ecl/hthor/hthor.ipp

@@ -2301,6 +2301,7 @@ public:
     virtual unsigned __int64 getFilePosition(const void * row);
     virtual unsigned __int64 getLocalFilePosition(const void * row);
     virtual const char * queryLogicalFilename(const void * row) { return logicalFileName.get(); }
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
 };
 
 class CHThorBinaryDiskReadBase : public CHThorDiskReadBaseActivity, implements IIndexReadContext

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1641,7 +1641,7 @@ bool CWsDfuEx::onDFURecordTypeInfo(IEspContext &context, IEspDFURecordTypeInfoRe
             df->queryAttributes().getPropBin("_rtlType", layoutBin);
             if (req.getIncludeJsonTypeInfo())
             {
-                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
                 const RtlTypeInfo *typeInfo = deserializer->deserialize(layoutBin);
                 StringBuffer jsonFormat;
                 dumpTypeInfo(jsonFormat, typeInfo);
@@ -2380,7 +2380,7 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
             df->queryAttributes().getPropBin("_rtlType", layoutBin);
             if (includeJsonTypeInfo)
             {
-                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
                 const RtlTypeInfo *typeInfo = deserializer->deserialize(layoutBin);
                 StringBuffer jsonFormat;
                 dumpTypeInfo(jsonFormat, typeInfo);

roxie/ccd/ccdactivities.cpp

@@ -1455,6 +1455,7 @@ public:
     {
         return reader->queryLogicalFilename(row);
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { throwUnexpected(); }
 
     virtual void match(IColumnProvider &entry, offset_t startOffset, offset_t endOffset)
     {

roxie/ccd/ccdfile.cpp

@@ -2544,10 +2544,10 @@ public:
                         diskTypeInfo.append(NULL);
                         break;
                     case 1:
-                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, false, nullptr));
+                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, false));
                         break;
                     case 2:
-                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, true, nullptr));
+                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, true));
                         break;
                     case 3:
                         assertex(fileNo > 0);

roxie/ccd/ccdkey.cpp

@@ -523,6 +523,8 @@ public:
     {
         deserializeSource.finishedRow();
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { throwUnexpected(); }
+
 };
 
 class InMemoryDirectReader : public CDirectReaderBase
@@ -1239,6 +1241,7 @@ public:
     { 
         UNIMPLEMENTED;
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
 
     virtual void serializeCursorPos(MemoryBuffer &mb) const 
     {

roxie/ccd/ccdserver.cpp

@@ -22173,6 +22173,7 @@ public:
     {
         return reader->queryLogicalFilename(row);
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
 
     virtual const void *nextRow()
     {

rtl/eclrtl/eclhelper_dyn.cpp

@@ -40,9 +40,9 @@
 class CDeserializedOutputMetaData : public COutputMetaData
 {
 public:
-    CDeserializedOutputMetaData(MemoryBuffer &binInfo, bool isGrouped, IThorIndexCallback *callback);
-    CDeserializedOutputMetaData(IPropertyTree &jsonInfo, bool isGrouped, IThorIndexCallback *callback);
-    CDeserializedOutputMetaData(const char *json, bool isGrouped, IThorIndexCallback *callback);
+    CDeserializedOutputMetaData(MemoryBuffer &binInfo, bool isGrouped);
+    CDeserializedOutputMetaData(IPropertyTree &jsonInfo, bool isGrouped);
+    CDeserializedOutputMetaData(const char *json, bool isGrouped);
     ~CDeserializedOutputMetaData();
 
     virtual const RtlTypeInfo * queryTypeInfo() const override { return typeInfo; }
@@ -54,25 +54,25 @@ protected:
 
 };
 
-CDeserializedOutputMetaData::CDeserializedOutputMetaData(MemoryBuffer &binInfo, bool isGrouped, IThorIndexCallback *callback)
+CDeserializedOutputMetaData::CDeserializedOutputMetaData(MemoryBuffer &binInfo, bool isGrouped)
 {
-    deserializer.setown(createRtlFieldTypeDeserializer(callback));
+    deserializer.setown(createRtlFieldTypeDeserializer());
     typeInfo = deserializer->deserialize(binInfo);
     if (isGrouped)
         flags |= MDFgrouped;
 }
 
-CDeserializedOutputMetaData::CDeserializedOutputMetaData(IPropertyTree &jsonInfo, bool isGrouped, IThorIndexCallback *callback)
+CDeserializedOutputMetaData::CDeserializedOutputMetaData(IPropertyTree &jsonInfo, bool isGrouped)
 {
-    deserializer.setown(createRtlFieldTypeDeserializer(callback));
+    deserializer.setown(createRtlFieldTypeDeserializer());
     typeInfo = deserializer->deserialize(jsonInfo);
     if (isGrouped)
         flags |= MDFgrouped;
 }
 
-CDeserializedOutputMetaData::CDeserializedOutputMetaData(const char *json, bool isGrouped, IThorIndexCallback *callback)
+CDeserializedOutputMetaData::CDeserializedOutputMetaData(const char *json, bool isGrouped)
 {
-    deserializer.setown(createRtlFieldTypeDeserializer(callback));
+    deserializer.setown(createRtlFieldTypeDeserializer());
     typeInfo = deserializer->deserialize(json);
     if (isGrouped)
         flags |= MDFgrouped;
@@ -85,19 +85,19 @@ CDeserializedOutputMetaData::~CDeserializedOutputMetaData()
     recordAccessor[0] = recordAccessor[1] = nullptr;
 }
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &binInfo, bool isGrouped, IThorIndexCallback *callback)
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &binInfo, bool isGrouped)
 {
-    return new CDeserializedOutputMetaData(binInfo, isGrouped, callback);
+    return new CDeserializedOutputMetaData(binInfo, isGrouped);
 }
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonInfo, bool isGrouped, IThorIndexCallback *callback)
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonInfo, bool isGrouped)
 {
-    return new CDeserializedOutputMetaData(jsonInfo, isGrouped, callback);
+    return new CDeserializedOutputMetaData(jsonInfo, isGrouped);
 }
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, bool isGrouped, IThorIndexCallback *callback)
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, bool isGrouped)
 {
-    return new CDeserializedOutputMetaData(json, isGrouped, callback);
+    return new CDeserializedOutputMetaData(json, isGrouped);
 }
 //---------------------------------------------------------------------------------------------------------------------
 
@@ -185,10 +185,14 @@ private:
 class ECLRTL_API CDynamicIndexReadArg : public CThorIndexReadArg, implements IDynamicIndexReadArg
 {
 public:
-    CDynamicIndexReadArg(const char *_fileName, IOutputMetaData *_in, IOutputMetaData *_projected, IOutputMetaData *_out, unsigned __int64 _chooseN, unsigned __int64 _skipN, unsigned __int64 _rowLimit)
-        : fileName(_fileName), in(_in), projected(_projected), out(_out), chooseN(_chooseN), skipN(_skipN), rowLimit(_rowLimit)
+    CDynamicIndexReadArg(const char *_fileName, IOutputMetaData *_in, IOutputMetaData *_projected, IOutputMetaData *_out,
+                         unsigned __int64 _chooseN, unsigned __int64 _skipN, unsigned __int64 _rowLimit, IVirtualFieldCallback &_callback)
+        : fileName(_fileName), in(_in), projected(_projected), out(_out), chooseN(_chooseN), skipN(_skipN), rowLimit(_rowLimit), callback(_callback)
     {
-        translator.setown(createRecordTranslator(out->queryRecordAccessor(true), projected->queryRecordAccessor(true)));
+        translator.setown(createRecordTranslator(out->queryRecordAccessor(true), in->queryRecordAccessor(true)));
+#ifdef _DEBUG
+        translator->describe();
+#endif
     }
     virtual bool needTransform() override final
     {
@@ -229,7 +233,7 @@ public:
     }
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * src) override final
     {
-        return translator->translate(rowBuilder, fieldCallback, (const byte *) src);
+        return translator->translate(rowBuilder, callback, (const byte *) src);
     }
     virtual unsigned __int64 getChooseNLimit() override final { return chooseN; }
     virtual unsigned __int64 getRowLimit() override final { return rowLimit; }
@@ -240,13 +244,13 @@ public:
     }
 
 private:
-    UnexpectedVirtualFieldCallback fieldCallback;
     StringAttr fileName;
     unsigned flags = 0;
     Owned<IOutputMetaData> in;
     Owned<IOutputMetaData> projected;
     Owned<IOutputMetaData> out;
     Owned<const IDynamicTransform> translator;
+    IVirtualFieldCallback &callback;
     RowFilter filters;
     unsigned __int64 chooseN = I64C(0x7fffffffffffffff); // constant(s) should be commoned up somewhere
     unsigned __int64 skipN = 0;
@@ -273,7 +277,7 @@ static IOutputMetaData *loadTypeInfo(IPropertyTree &xgmml, const char *key)
     assertex(binInfo.length());
     bool grouped = xgmml.getPropBool(xpath.setf("att[@name='%s_binary']/value", key), false);
 
-    return new CDeserializedOutputMetaData(binInfo, grouped, nullptr);
+    return new CDeserializedOutputMetaData(binInfo, grouped);
 }
 
 
@@ -282,9 +286,9 @@ extern ECLRTL_API IHThorDiskReadArg *createDiskReadArg(const char *fileName, IOu
     return new CDynamicDiskReadArg(fileName, in, projected, out, chooseN, skipN, rowLimit);
 }
 
-extern ECLRTL_API IHThorIndexReadArg *createIndexReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *projected, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit)
+extern ECLRTL_API IHThorIndexReadArg *createIndexReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *projected, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit, IVirtualFieldCallback &callback)
 {
-    return new CDynamicIndexReadArg(fileName, in, projected, out, chooseN, skipN, rowLimit);
+    return new CDynamicIndexReadArg(fileName, in, projected, out, chooseN, skipN, rowLimit, callback);
 }
 
 

rtl/eclrtl/eclhelper_dyn.hpp

@@ -22,9 +22,9 @@
 #include "eclrtl.hpp"
 #include "eclhelper.hpp"
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &mb, bool isGroupedPersist, IThorIndexCallback *callback);
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonTree, bool isGroupedPersist, IThorIndexCallback *callback);
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, bool isGroupedPersist, IThorIndexCallback *callback);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &mb, bool isGroupedPersist);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonTree, bool isGroupedPersist);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, bool isGroupedPersist);
 
 interface IDynamicIndexReadArg
 {
@@ -32,7 +32,7 @@ interface IDynamicIndexReadArg
 };
 
 extern ECLRTL_API IHThorDiskReadArg *createDiskReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *projected, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit);
-extern ECLRTL_API IHThorIndexReadArg *createIndexReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *projecte, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit);
+extern ECLRTL_API IHThorIndexReadArg *createIndexReadArg(const char *fileName, IOutputMetaData *in, IOutputMetaData *projecte, IOutputMetaData *out, unsigned __int64 chooseN, unsigned __int64 skipN, unsigned __int64 rowLimit, IVirtualFieldCallback &callback);
 extern ECLRTL_API IEclProcess* createDynamicEclProcess();
 
 #endif

rtl/eclrtl/rtldynfield.cpp

@@ -30,7 +30,7 @@
 //#define TRACE_TRANSLATION
 #define VALIDATE_TYPEINFO_HASHES
 
-#define RTLTYPEINFO_FORMAT_1   80   // In case we ever want to support more than one format
+#define RTLTYPEINFO_FORMAT_1   81   // In case we ever want to support more than one format or change how it is stored
 
 //---------------------------------------------------------------------------------------------------------------------
 
@@ -61,7 +61,7 @@ extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val)
 
 //---------------------------------------------------------------------------------------------------------------------
 
-const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_callback) const
+const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo() const
 {
     const RtlTypeInfo *ret = nullptr;
     switch (fieldType & RFTMkind)
@@ -72,15 +72,12 @@ const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_c
     case type_keyedint:
         ret = new RtlKeyedIntTypeInfo(fieldType, length, childType);
         break;
-    case type_blob:  // MORE - will need its own type (see code below)
     case type_int:
         ret = new RtlIntTypeInfo(fieldType, length);
         break;
-#if 0 // Later when implemented
     case type_blob:
-        ret = new RtlBlobTypeInfo(fieldType, length, childType, _callback);
+        ret = new RtlBlobTypeInfo(fieldType, length, childType);
         break;
-#endif
     case type_filepos:
 #if __BYTE_ORDER == __LITTLE_ENDIAN
         ret = new RtlSwapIntTypeInfo(fieldType, length);
@@ -515,8 +512,7 @@ public:
      *
      * @param  _callback Supplies a callback to be used for blobs/filepositions.
      */
-    CRtlFieldTypeDeserializer(IThorIndexCallback *_callback)
-    : callback(_callback)
+    CRtlFieldTypeDeserializer()
     {
     }
     /**
@@ -645,7 +641,7 @@ public:
             return *found;
         savedTypes.append(LINK(typeOrIfblock));
         info.locale = keep(info.locale);
-        const RtlTypeInfo * ret = info.createRtlTypeInfo(callback);
+        const RtlTypeInfo * ret = info.createRtlTypeInfo();
         types.setValue(name, ret);
         unsigned baseType = (info.fieldType & RFTMkind);
         if (baseType == type_record)
@@ -672,7 +668,6 @@ private:
     KeptAtomTable atoms;     // Used to ensure proper lifetime of strings used in type structures
     MapStringTo<const RtlTypeInfo *> types;  // Ensures structures only generated once
     const RtlTypeInfo *base = nullptr;       // Holds the resulting type
-    IThorIndexCallback *callback = nullptr;
     IConstPointerArray savedTypes; // ensure types remain alive for subsequent lookups
     void deleteType(const RtlTypeInfo *type)
     {
@@ -788,7 +783,7 @@ private:
             info.filter = deserializeFieldFilter(fieldId, *fieldType, filterText);
         }
 
-        const RtlTypeInfo * result = info.createRtlTypeInfo(callback);
+        const RtlTypeInfo * result = info.createRtlTypeInfo();
         if (baseType == type_record)
             patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
         return result;
@@ -861,31 +856,12 @@ private:
             }
         }
         info.fieldType &= ~RFTMserializerFlags;
-        const RtlTypeInfo * result = info.createRtlTypeInfo(callback);
+        const RtlTypeInfo * result = info.createRtlTypeInfo();
         if (baseType == type_record)
             patchIfBlockParentRow(result, static_cast<const RtlRecordTypeInfo *>(result));
         return result;
     }
-    void patchIndexFilePos()
-    {
-        if (callback && (base->fieldType & RFTMkind) == type_record)
-        {
-            // Yukky hack time
-            // Assumes that the fieldinfo is not shared...
-            // But that is also assumed by the code that cleans them up.
-            const RtlFieldInfo * const *fields = base->queryFields();
-            for(;;)
-            {
-                const RtlFieldInfo *field = *fields++;
-                if (!field)
-                    break;
-                if (field->type->getType() == type_blob)
-                {
-                    static_cast<RtlBlobTypeInfo *>(const_cast<RtlTypeInfo *>(field->type))->setCallback(callback);
-                }
-            }
-        }
-    }
+
     void patchIfBlockParentRow(const RtlTypeInfo * fieldType, const RtlRecordTypeInfo * parentRow)
     {
         const RtlFieldInfo * const * fields = fieldType->queryFields();
@@ -907,9 +883,9 @@ private:
     }
 };
 
-extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback)
+extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer()
 {
-    return new CRtlFieldTypeDeserializer(callback);
+    return new CRtlFieldTypeDeserializer();
 }
 
 extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t)
@@ -956,12 +932,12 @@ extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,I
 
 #ifdef _DEBUG
         StringBuffer ret2;
-        CRtlFieldTypeDeserializer deserializer(nullptr);
+        CRtlFieldTypeDeserializer deserializer;
         CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
         assert(streq(ret, ret2));
         MemoryBuffer out;
         CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
-        CRtlFieldTypeDeserializer bindeserializer(nullptr);
+        CRtlFieldTypeDeserializer bindeserializer;
         CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
         assert(streq(ret, ret2));
 #endif
@@ -1016,6 +992,7 @@ enum FieldMatchType {
 
     // This flag may be set in conjunction with the others
     match_inifblock   = 0x400,   // matching to a field in an ifblock - may not be present
+    match_deblob      = 0x1000,  // source needs fetching from a blob prior to translation
 };
 
 StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
@@ -1035,6 +1012,7 @@ StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
     if (flags & match_keychange) out.append("|keychange");
     if (flags & match_fail) out.append("|fail");
     if (flags & match_virtual) out.append("|virtual");
+    if (flags & match_deblob) out.append("|blob");
     assertex(out.length() > origlen);
     return out.remove(origlen, 1);
 }
@@ -1050,6 +1028,9 @@ public:
     {
         matchInfo = new MatchInfo[destRecInfo.getNumFields()];
         createMatchInfo();
+#ifdef _DEBUG
+        //describe();
+#endif
     }
     ~GeneralRecordTranslator()
     {
@@ -1126,6 +1107,7 @@ private:
         byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
         memset(destConditions, 2, destRecInfo.getNumIfBlocks() * sizeof(byte));
         size32_t estimate = destRecInfo.getFixedSize();
+        bool hasBlobs = false;
         if (!estimate)
         {
             estimate = estimateNewSize(sourceRow);
@@ -1170,29 +1152,41 @@ private:
                 size_t sourceOffset = sourceRow.getOffset(matchField);
                 const byte *source = sourceRow.queryRow() + sourceOffset;
                 size_t copySize = sourceRow.getSize(matchField);
+                if (match.matchType & match_deblob)
+                {
+                    offset_t blobId = sourceType->getInt(source);
+                    sourceType = sourceType->queryChildType();
+                    sourceOffset = 0;
+                    source = callback.lookupBlob(blobId);
+                    copySize = sourceType->size(source, source);
+                    hasBlobs = true;
+                }
                 if (copySize == 0 && (match.matchType & match_inifblock))  // Field is missing because of an ifblock - use default value
                 {
                     offset = type->buildNull(builder, offset, field);
                 }
                 else
                 {
-                    switch (match.matchType & ~match_inifblock)
+                    switch (match.matchType & ~(match_inifblock|match_deblob))
                     {
                     case match_perfect:
                     {
                         // Look ahead for other perfect matches and combine the copies
-                        while (idx < destRecInfo.getNumFields()-1)
+                        if (!(match.matchType & match_deblob))
                         {
-                            const MatchInfo &nextMatch = matchInfo[idx+1];
-                            if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
+                            while (idx < destRecInfo.getNumFields()-1)
                             {
-                                idx++;
-                                matchField++;
+                                const MatchInfo &nextMatch = matchInfo[idx+1];
+                                if (nextMatch.matchType == match_perfect && nextMatch.matchIdx == matchField+1)
+                                {
+                                    idx++;
+                                    matchField++;
+                                }
+                                else
+                                    break;
                             }
-                            else
-                                break;
+                            copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
                         }
-                        size_t copySize = sourceRow.getOffset(matchField+1) - sourceOffset;
                         builder.ensureCapacity(offset+copySize, field->name);
                         memcpy(builder.getSelf()+offset, source, copySize);
                         offset += copySize;
@@ -1201,7 +1195,7 @@ private:
                     case match_truncate:
                     {
                         assert(type->isFixedSize());
-                        size32_t copySize = type->getMinSize();
+                        copySize = type->getMinSize();
                         builder.ensureCapacity(offset+copySize, field->name);
                         memcpy(builder.getSelf()+offset, source, copySize);
                         offset += copySize;
@@ -1327,7 +1321,7 @@ private:
             {
                 // Note - ifblocks make this assertion invalid. We do not account for potentially omitted fields
                 // when estimating target record size.
-                if (!destRecInfo.getNumIfBlocks())
+                if (!destRecInfo.getNumIfBlocks() && !hasBlobs)
                     assert(offset-origOffset > estimate);  // Estimate is always supposed to be conservative
     #ifdef TRACE_TRANSLATION
                 DBGLOG("Wrote %u bytes to record (estimate was %u)\n", offset-origOffset, estimate);
@@ -1420,7 +1414,19 @@ private:
             }
             else
             {
+                bool deblob = false;
                 const RtlTypeInfo *sourceType = sourceRecInfo.queryType(info.matchIdx);
+                if (sourceType->isBlob())
+                {
+                    if (type->isBlob())
+                    {
+                    }
+                    else
+                    {
+                        sourceType = sourceType->queryChildType();
+                        deblob = true;
+                    }
+                }
                 if (!type->isScalar() || !sourceType->isScalar())
                 {
                     if (type->getType() != sourceType->getType())
@@ -1473,6 +1479,12 @@ private:
                                 info.matchType = match_fail;
                             break;
                         }
+                        case type_blob:
+                            if (sourceType->isBlob())
+                                info.matchType = match_perfect;  // We don't check that the child type matches
+                            else
+                                info.matchType = match_fail;
+                            break;
                         default:
                             info.matchType = match_fail;
                             break;
@@ -1511,6 +1523,8 @@ private:
                 }
                 else
                     info.matchType = match_typecast;
+                if (deblob)
+                    info.matchType |= match_deblob;
                 unsigned sourceFlags = sourceRecInfo.queryField(info.matchIdx)->flags;
                 if (sourceFlags & RFTMinifblock)
                     info.matchType |= match_inifblock;  // Avoids incorrect commoning up of adjacent matches
@@ -1766,6 +1780,11 @@ unsigned __int64 NullVirtualFieldCallback::getLocalFilePosition(const void * row
     return 0;
 }
 
+const byte * NullVirtualFieldCallback::lookupBlob(unsigned __int64 id)
+{
+    return nullptr;
+}
+
 const char * UnexpectedVirtualFieldCallback::queryLogicalFilename(const void * row)
 {
     throwUnexpectedX("VIRTUAL(LOGICALFILENAME)");
@@ -1781,6 +1800,11 @@ unsigned __int64 UnexpectedVirtualFieldCallback::getLocalFilePosition(const void
     throwUnexpectedX("VIRTUAL(LOCALFILEPOSITION)");
 }
 
+const byte * UnexpectedVirtualFieldCallback::lookupBlob(unsigned __int64 id)
+{
+    throwUnexpectedX("BLOB");
+}
+
 unsigned __int64 FetchVirtualFieldCallback::getFilePosition(const void * row)
 {
     return filepos;
@@ -1801,3 +1825,8 @@ unsigned __int64 LocalVirtualFieldCallback::getLocalFilePosition(const void * ro
     return localfilepos;
 }
 
+const byte * LocalVirtualFieldCallback::lookupBlob(unsigned __int64 id)
+{
+    throwUnexpectedX("BLOB");
+}
+

rtl/eclrtl/rtldynfield.hpp

@@ -39,7 +39,7 @@ public:
     const RtlFieldInfo * * fieldsArray = nullptr;
     const IFieldFilter * filter = nullptr;
 
-    const RtlTypeInfo *createRtlTypeInfo(IThorIndexCallback *_callback) const;
+    const RtlTypeInfo *createRtlTypeInfo() const;
 };
 
 interface ITypeInfo;
@@ -153,7 +153,7 @@ extern ECLRTL_API void throwTranslationError(const RtlRecord &_destRecInfo, cons
 
 extern ECLRTL_API const IKeyTranslator *createKeyTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo);
 
-extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback);
+extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer();
 
 extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t);
 
@@ -194,6 +194,7 @@ public:
     virtual const char * queryLogicalFilename(const void * row) override;
     virtual unsigned __int64 getFilePosition(const void * row) override;
     virtual unsigned __int64 getLocalFilePosition(const void * row) override;
+    virtual const byte * lookupBlob(unsigned __int64 id) override;
 };
 
 class ECLRTL_API UnexpectedVirtualFieldCallback : public CInterfaceOf<IVirtualFieldCallback>
@@ -202,6 +203,8 @@ public:
     virtual const char * queryLogicalFilename(const void * row) override;
     virtual unsigned __int64 getFilePosition(const void * row) override;
     virtual unsigned __int64 getLocalFilePosition(const void * row) override;
+    virtual const byte * lookupBlob(unsigned __int64 id) override;
+
 };
 
 typedef UnexpectedVirtualFieldCallback IndexVirtualFieldCallback;
@@ -226,6 +229,7 @@ public:
     virtual const char * queryLogicalFilename(const void * row) override;
     virtual unsigned __int64 getFilePosition(const void * row) override;
     virtual unsigned __int64 getLocalFilePosition(const void * row) override;
+    virtual const byte * lookupBlob(unsigned __int64 id) override;
 private:
     const char * filename;
     unsigned __int64 filepos;

rtl/eclrtl/rtlfield.cpp

@@ -614,19 +614,16 @@ size32_t RtlBlobTypeInfo::buildUtf8(ARowBuilder &builder, size32_t offset, const
 
 size32_t RtlBlobTypeInfo::process(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IFieldProcessor & target) const
 {
-    assertex(callback);
     UNIMPLEMENTED;
 }
 
 size32_t RtlBlobTypeInfo::toXML(const byte * self, const byte * selfrow, const RtlFieldInfo * field, IXmlWriter & target) const
 {
-    assertex(callback);
     UNIMPLEMENTED;
 }
 
 void RtlBlobTypeInfo::getString(size32_t & resultLen, char * & result, const void * ptr) const
 {
-    assertex(callback);
     UNIMPLEMENTED;
 }
 
@@ -637,7 +634,7 @@ void RtlBlobTypeInfo::getUtf8(size32_t & resultLen, char * & result, const void
 
 __int64 RtlBlobTypeInfo::getInt(const void * ptr) const
 {
-    UNIMPLEMENTED;
+    return *(offset_t *) ptr;
 }
 
 double RtlBlobTypeInfo::getReal(const void * ptr) const
@@ -655,11 +652,6 @@ bool RtlBlobTypeInfo::canExtend(char &fillChar) const
     return false;
 }
 
-void RtlBlobTypeInfo::setCallback(IThorIndexCallback *_callback)
-{
-    callback = _callback;
-}
-
 int RtlBlobTypeInfo::compare(const byte * left, const byte * right) const
 {
     UNIMPLEMENTED;

rtl/eclrtl/rtlfield.hpp

@@ -143,12 +143,9 @@ struct ECLRTL_API RtlIntTypeInfo : public RtlTypeInfoBase
 
 struct ECLRTL_API RtlBlobTypeInfo : public RtlTypeInfoBase
 {
-    // Used for values stored in the fieldpos field of indexes
-    constexpr inline RtlBlobTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo *_child, IThorIndexCallback *_callback)
-    : RtlTypeInfoBase(_fieldType, _length), child(_child), callback(_callback)
-    {}
+    // Used for values stored in blobs. The child type represents the original type
     constexpr inline RtlBlobTypeInfo(unsigned _fieldType, unsigned _length, const RtlTypeInfo *_child)
-    : RtlTypeInfoBase(_fieldType, _length), child(_child), callback(nullptr)
+    : RtlTypeInfoBase(_fieldType, _length), child(_child)
     {}
     virtual void doDelete() const final override { delete this; }
 
@@ -166,15 +163,13 @@ struct ECLRTL_API RtlBlobTypeInfo : public RtlTypeInfoBase
     virtual double getReal(const void * ptr) const override;
     virtual bool canTruncate() const override;
     virtual bool canExtend(char &fillChar) const override;
-    virtual bool isNumeric() const override { return true; }
+    virtual bool isNumeric() const override { return false; }
+    virtual bool isScalar() const override { return false; }
     virtual int compare(const byte * left, const byte * right) const override;
     virtual unsigned hash(const byte *self, unsigned inhash) const override;
     virtual const RtlTypeInfo * queryChildType() const override { return child; }
-
-    void setCallback(IThorIndexCallback *_callback);
 private:
     const RtlTypeInfo *child = nullptr;
-    IThorIndexCallback *callback = nullptr;
 };
 
 struct ECLRTL_API RtlSwapIntTypeInfo : public RtlTypeInfoBase

rtl/eclrtl/rtlnewkey.cpp

@@ -1784,7 +1784,7 @@ IFieldFilter * createSubStringFieldFilter(unsigned fieldId, size32_t subLength,
     FieldTypeInfoStruct info;
     info.fieldType = (type.fieldType & ~RFTMunknownsize);
     info.length = subLength;
-    Owned<SharedRtlTypeInfo> subType(new SharedRtlTypeInfo(info.createRtlTypeInfo(nullptr)));
+    Owned<SharedRtlTypeInfo> subType(new SharedRtlTypeInfo(info.createRtlTypeInfo()));
 
     //Create a new set of values truncated to the appropriate length.
     Owned<IValueSet> newValues = values->cast(*subType->type);
@@ -1932,7 +1932,7 @@ IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type
             FieldTypeInfoStruct info;
             info.fieldType = (type.fieldType & ~RFTMunknownsize);
             info.length = subLength;
-            Owned<SharedRtlTypeInfo> subType(new SharedRtlTypeInfo(info.createRtlTypeInfo(nullptr)));
+            Owned<SharedRtlTypeInfo> subType(new SharedRtlTypeInfo(info.createRtlTypeInfo()));
 
             //The serialized set is already truncated to the appropriate length.
             Owned<IValueSet> values = createValueSet(*subType->type);
@@ -1995,7 +1995,7 @@ IFieldFilter * deserializeFieldFilter(unsigned fieldId, const RtlTypeInfo & type
             FieldTypeInfoStruct info;
             info.fieldType = (type.fieldType & ~RFTMunknownsize);
             info.length = subLength;
-            Owned<SharedRtlTypeInfo> subType(new SharedRtlTypeInfo(info.createRtlTypeInfo(nullptr)));
+            Owned<SharedRtlTypeInfo> subType(new SharedRtlTypeInfo(info.createRtlTypeInfo()));
 
             //The serialized set is already truncated to the appropriate length.
             Owned<IValueSet> values = createValueSet(*subType->type, in);
@@ -2035,13 +2035,14 @@ void RowFilter::addFilter(const IFieldFilter & filter)
         numFieldsRequired = fieldNum+1;
 }
 
-void RowFilter::addFilter(const RtlRecord & record, const char * filterText)
+const IFieldFilter & RowFilter::addFilter(const RtlRecord & record, const char * filterText)
 {
     IFieldFilter & filter = *deserializeFieldFilter(record, filterText);
     filters.append(filter);
     unsigned fieldNum = filter.queryFieldIndex();
     if (fieldNum >= numFieldsRequired)
         numFieldsRequired = fieldNum+1;
+    return filter;
 }
 
 bool RowFilter::matches(const RtlRow & row) const

rtl/eclrtl/rtlnewkey.hpp

@@ -58,7 +58,7 @@ class ECLRTL_API RowFilter
 {
 public:
     void addFilter(const IFieldFilter & filter);
-    void addFilter(const RtlRecord & record, const char * filter);
+    const IFieldFilter &addFilter(const RtlRecord & record, const char * filter);
     bool matches(const RtlRow & row) const;
 
     void createSegmentMonitors(IIndexReadContext *irc);

rtl/include/eclhelper.hpp

@@ -426,6 +426,7 @@ struct RtlTypeInfo : public RtlITypeInfo
     inline bool isLinkCounted() const { return (fieldType & RFTMlinkcounted) != 0; }
     inline bool isSigned() const { return (fieldType & RFTMunsigned) == 0; }
     inline bool isUnsigned() const { return (fieldType & RFTMunsigned) != 0; }
+    inline bool isBlob() const { return getType() == type_blob; }
     inline unsigned getDecimalDigits() const { return (length & 0xffff); }
     inline unsigned getDecimalPrecision() const { return (length >> 16); }
     inline unsigned getBitfieldIntSize() const { return (length & 0xff); }
@@ -2302,6 +2303,7 @@ interface ISteppingMeta
 interface IThorDiskCallback : extends IFilePositionProvider
 {
     virtual const char * queryLogicalFilename(const void * row) = 0;
+    virtual const byte * lookupBlob(unsigned __int64 id) = 0;         // return reference, not freed by code generator, can dispose once transform() has returned.
 };
 
 interface IThorIndexCallback : extends IInterface

system/jhtree/jhtree.cpp

@@ -2084,6 +2084,7 @@ void IndexRowFilter::finish(size32_t _keyedSize)
         unsigned idx = numFilterFields();
         append(FFkeyed, createWildFieldFilter(idx, *recInfo.queryType(idx)));
     }
+    assertex(numFilterFields() == keySegCount);
 }
 
 void IndexRowFilter::describe(StringBuffer &out) const
@@ -3018,7 +3019,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
                                " { \"name\": \"f1\", \"type\": \"ty1\", \"flags\": 4 }, "
                                " { \"name\": \"f2\", \"type\": \"ty2\", \"flags\": 4 } ] "
                                "}";
-            Owned<IOutputMetaData> meta = createTypeInfoOutputMetaData(json, false, nullptr);
+            Owned<IOutputMetaData> meta = createTypeInfoOutputMetaData(json, false);
             Owned <IKeyManager> tlk1 = createKeyMerger(meta->queryRecordAccessor(true), keyset, 7, NULL, false);
             Owned<IStringSet> sset1 = createStringSet(7);
             sset1->addRange("0000003", "0000003");
@@ -3210,7 +3211,7 @@ protected:
                 " { \"name\": \"f1\", \"type\": \"ty1\", \"flags\": 4 }, "
                 " ] "
                 "}";
-        Owned<IOutputMetaData> meta = createTypeInfoOutputMetaData(json, false, nullptr);
+        Owned<IOutputMetaData> meta = createTypeInfoOutputMetaData(json, false);
         const RtlRecord &recInfo = meta->queryRecordAccessor(true);
         buildTestKeys(variable);
         {

thorlcr/activities/thdiskbaseslave.ipp

@@ -62,6 +62,7 @@ public:
     virtual offset_t getFilePosition(const void * row);
     virtual offset_t getLocalFilePosition(const void * row);
     virtual const char * queryLogicalFilename(const void * row);
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
 
 // CPartHandler
     virtual void setPart(IPartDescriptor *_partDesc);

thorlcr/slave/slavmain.cpp

@@ -1293,12 +1293,12 @@ public:
                         {
                             unsigned publishedFormatCrc;
                             msg.read(publishedFormatCrc);
-                            Owned<IOutputMetaData> publishedFormat = createTypeInfoOutputMetaData(msg, false, nullptr);
+                            Owned<IOutputMetaData> publishedFormat = createTypeInfoOutputMetaData(msg, false);
                             Owned<IOutputMetaData> projectedFormat;
                             bool projected;
                             msg.read(projected);
                             if (projected)
-                                projectedFormat.setown(createTypeInfoOutputMetaData(msg, false, nullptr));
+                                projectedFormat.setown(createTypeInfoOutputMetaData(msg, false));
                             else
                                 projectedFormat.set(publishedFormat);
                             if (created) // translation for the key context will already have been setup and do not want to free existing
@@ -1365,12 +1365,12 @@ public:
                         {
                             unsigned publishedFormatCrc;
                             msg.read(publishedFormatCrc);
-                            Owned<IOutputMetaData> publishedFormat = createTypeInfoOutputMetaData(msg, false, nullptr);
+                            Owned<IOutputMetaData> publishedFormat = createTypeInfoOutputMetaData(msg, false);
                             Owned<IOutputMetaData> projectedFormat;
                             bool projected;
                             msg.read(projected);
                             if (projected)
-                                projectedFormat.setown(createTypeInfoOutputMetaData(msg, false, nullptr));
+                                projectedFormat.setown(createTypeInfoOutputMetaData(msg, false));
                             else
                                 projectedFormat.set(publishedFormat);
 

tools/dumpkey/CMakeLists.txt

@@ -30,10 +30,14 @@ set (    SRCS
     )
 
 include_directories ( 
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
          ./../../system/jhtree 
          ./../../rtl/include
          ./../../rtl/eclrtl
          ./../../system/include 
+         ./../../ecl/hql 
+         ./../../common/deftype
          ./../../common/thorhelper 
          ./../../system/jlib 
     )
@@ -48,6 +52,7 @@ target_link_libraries ( dumpkey
          thorhelper
          jlib
          jhtree
+         hql
     )
 
 

tools/dumpkey/dumpkey.cpp

@@ -20,7 +20,10 @@
 #include "ctfile.hpp"
 #include "rtlrecord.hpp"
 #include "rtlformat.hpp"
+#include "rtldynfield.hpp"
 #include "eclhelper_dyn.hpp"
+#include "hqlexpr.hpp"
+#include "hqlutil.hpp"
 
 void fatal(const char *format, ...) __attribute__((format(printf, 1, 2)));
 void fatal(const char *format, ...)
@@ -76,6 +79,33 @@ void doOption(const char *opt)
         usage();
 }
 
+class MyIndexVirtualFieldCallback : public CInterfaceOf<IVirtualFieldCallback>
+{
+public:
+    MyIndexVirtualFieldCallback(IKeyManager *_manager) : manager(_manager)
+    {
+    }
+    virtual const char * queryLogicalFilename(const void * row) override
+    {
+        UNIMPLEMENTED;
+    }
+    virtual unsigned __int64 getFilePosition(const void * row) override
+    {
+        UNIMPLEMENTED;
+    }
+    virtual unsigned __int64 getLocalFilePosition(const void * row) override
+    {
+        UNIMPLEMENTED;
+    }
+    virtual const byte * lookupBlob(unsigned __int64 id) override
+    {
+        size32_t blobSize;
+        return manager->loadBlob(id, blobSize);
+    }
+private:
+    Linked<IKeyManager> manager;
+};
+
 int main(int argc, const char **argv)
 {
     InitModuleObjects();
@@ -116,6 +146,7 @@ int main(int argc, const char **argv)
             const char * keyName = files.item(idx);
             index.setown(createKeyIndex(keyName, 0, false, false));
             size32_t key_size = index->keySize();  // NOTE - in variable size case, this is 32767
+            size32_t keyedSize = index->keyedSize();
             unsigned nodeSize = index->getNodeSize();
             if (optFullHeader)
             {
@@ -128,7 +159,7 @@ int main(int argc, const char **argv)
                 io->read(0, sizeof(KeyHdr), (void *)block.get());
                 header->load(*(KeyHdr*)block.get());
 
-                printf("Key '%s'\nkeySize=%d NumParts=%x, Top=%d\n", keyName, key_size, index->numParts(), index->isTopLevelKey());
+                printf("Key '%s'\nkeySize=%d keyedSize = %d NumParts=%x, Top=%d\n", keyName, key_size, keyedSize, index->numParts(), index->isTopLevelKey());
                 printf("File size = %" I64F "d, nodes = %" I64F "d\n", in->size(), in->size() / nodeSize - 1);
                 printf("rootoffset=%" I64F "d[%" I64F "d]\n", header->getRootFPos(), header->getRootFPos()/nodeSize);
                 Owned<IPropertyTree> metadata = index->getMetadata();
@@ -171,30 +202,41 @@ int main(int argc, const char **argv)
                 ArrayOf<const RtlFieldInfo *> deleteFields;
                 ArrayOf<const RtlFieldInfo *> fields;  // Note - the lifetime of the array needs to extend beyond the lifetime of outmeta. The fields themselves are shared with diskmeta, and do not need to be released.
                 Owned<IOutputMetaData> outmeta;
-                Owned<IHThorIndexReadArg> helper;
                 Owned<IXmlWriterExt> writer;
-                class MyIndexCallback : public CInterfaceOf<IThorIndexCallback>
-                {
-                public:
-                    MyIndexCallback() {}
-                    virtual byte * lookupBlob(unsigned __int64 id) override
-                    {
-                        UNIMPLEMENTED;
-                    }
-                } callback;
+                Owned<const IDynamicTransform> translator;
+                RowFilter rowFilter;
                 unsigned __int64 count = globals->getPropInt("recs", 1);
                 const RtlRecordTypeInfo *outRecType = nullptr;
                 if (metadata && metadata->hasProp("_rtlType"))
                 {
                     MemoryBuffer layoutBin;
                     metadata->getPropBin("_rtlType", layoutBin);
-                    diskmeta.setown(createTypeInfoOutputMetaData(layoutBin, false, &callback));
+                    try
+                    {
+                        diskmeta.setown(createTypeInfoOutputMetaData(layoutBin, false));
+                    }
+                    catch (IException *E)
+                    {
+                        EXCLOG(E);
+                        E->Release();
+                    }
+                }
+                if (!diskmeta && metadata->hasProp("_record_ECL"))
+                {
+                    MultiErrorReceiver errs;
+                    Owned<IHqlExpression> expr = parseQuery(metadata->queryProp("_record_ECL"), &errs);
+                    if (errs.errCount() == 0)
+                    {
+                        MemoryBuffer layoutBin;
+                        if (exportBinaryType(layoutBin, expr, true))
+                            diskmeta.setown(createTypeInfoOutputMetaData(layoutBin, false));
+                    }
                 }
                 if (diskmeta)
                 {
                     writer.setown(new SimpleOutputWriter);
                     const RtlRecord &inrec = diskmeta->queryRecordAccessor(true);
-                    manager.setown(createLocalKeyManager(inrec, index, nullptr, false));
+                    manager.setown(createLocalKeyManager(inrec, index, nullptr, true));
                     size32_t minRecSize = 0;
                     if (globals->hasProp("fields"))
                     {
@@ -210,7 +252,6 @@ int main(int argc, const char **argv)
                             {
                                 // We can't just use the original source field in this case (as blobs are only supported in the input)
                                 // So instead, create a field in the target with the original type.
-                                //MORE: I'm not sure what this should do for a blob..., revisit when blobs are implemented
                                 field = new RtlFieldStrInfo(field->name, field->xpath, field->type->queryChildType());
                                 deleteFields.append(field);
                             }
@@ -234,24 +275,23 @@ int main(int argc, const char **argv)
                             fields.append(field);
                             minRecSize += field->type->getMinSize();
                         }
-                        outmeta.set(diskmeta);
                     }
                     fields.append(nullptr);
                     outRecType = new RtlRecordTypeInfo(type_record, minRecSize, fields.getArray(0));
                     outmeta.setown(new CDynamicOutputMetaData(*outRecType));
-                    helper.setown(createIndexReadArg(keyName, diskmeta.getLink(), outmeta.getLink(), outmeta.getLink(), count, 0, (uint64_t) -1));
-                    helper->setCallback(&callback);
+                    translator.setown(createRecordTranslator(outmeta->queryRecordAccessor(true), inrec));
                     if (filters.ordinality())
                     {
-                        IDynamicIndexReadArg *arg = QUERYINTERFACE(helper.get(), IDynamicIndexReadArg);
-                        assertex(arg);
                         ForEachItemIn(idx, filters)
                         {
-                            arg->addFilter(filters.item(idx));
+                            const IFieldFilter &thisFilter = rowFilter.addFilter(diskmeta->queryRecordAccessor(true), filters.item(idx));
+                            unsigned idx = thisFilter.queryFieldIndex();
+                            const RtlFieldInfo *field = inrec.queryOriginalField(idx);
+                            if (field->flags & RFTMispayloadfield)
+                                throw MakeStringException(0, "Cannot filter on payload field '%s'", field->name);
                         }
                     }
-                    helper->createSegmentMonitors(manager);
-                    count = helper->getChooseNLimit(); // Just because this is testing out the createIndexReadArg functionality
+                    rowFilter.createSegmentMonitors(manager);
                 }
                 else
                 {
@@ -261,6 +301,7 @@ int main(int argc, const char **argv)
                 }
                 manager->finishSegmentMonitors();
                 manager->reset();
+                MyIndexVirtualFieldCallback callback(manager);
                 while (manager->lookup(true) && count--)
                 {
                     byte const * buffer = manager->queryKeyBuffer();
@@ -276,26 +317,22 @@ int main(int argc, const char **argv)
                             printf("%02x", ((unsigned char) buffer[i]) & 0xff);
                         printf("  :%" I64F "u\n", seq);
                     }
-                    else if (helper)
+                    else if (translator)
                     {
-                        if (helper->canMatch(buffer))
+                        MemoryBuffer buf;
+                        MemoryBufferBuilder aBuilder(buf, 0);
+                        if (translator->translate(aBuilder, callback, buffer))
                         {
-                            MemoryBuffer buf;
-                            MemoryBufferBuilder aBuilder(buf, 0);
-                            if (helper->transform(aBuilder, (const byte *) buffer))
-                            {
-                                outmeta->toXML((const byte *) buf.toByteArray(), *writer.get());
-                                printf("%s\n", writer->str());
-                                writer->clear();
-                            }
-                            else
-                                count++;  // Don't count this row as it was postfiltered
+                            outmeta->toXML((const byte *) buf.toByteArray(), *writer.get());
+                            printf("%s\n", writer->str());
+                            writer->clear();
                         }
                         else
-                            count++;
+                            count++;  // Row was postfiltered
                     }
                     else
                         printf("%.*s  :%" I64F "u\n", size, buffer, seq);
+                    manager->releaseBlobs();
                 }
                 if (outRecType)
                     outRecType->doDelete();