HPCC-19552 Create correct type information for old indexes

If index does not have _rtlType information stored, use the ECL definition but
patch it up to get special keyed types correct.

Note that we cannot tell the difference - from the available information -
between indexes with FILEPOSITION(FALSE) and ones without it, in every case.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorcommon.cpp

@@ -31,6 +31,7 @@
 #include "rtldynfield.hpp"
 #include "eclhelper_dyn.hpp"
 #include "hqlexpr.hpp"
+#include "hqlutil.hpp"
 #include <algorithm>
 #ifdef _USE_NUMA
 #include <numa.h>
@@ -2010,19 +2011,21 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
         else if (props.hasProp("ECL"))
         {
             const char *kind = props.queryProp("@kind");
-            if (kind && streq(kind, "key"))
-            {
-                DBGLOG("Cannot deserialize file metadata: index too old");
-                return nullptr;
-            }
+            bool isIndex = (kind && streq(kind, "key"));
             StringBuffer layoutECL;
             props.getProp("ECL", layoutECL);
             MultiErrorReceiver errs;
             Owned<IHqlExpression> expr = parseQuery(layoutECL.str(), &errs);
             if (errs.errCount() == 0)
             {
+                if (props.hasProp("_record_layout"))  // Some old indexes need the payload count patched in from here
+                {
+                    MemoryBuffer mb;
+                    props.getPropBin("_record_layout", mb);
+                    expr.setown(patchEclRecordDefinitionFromRecordLayout(expr, mb));
+                }
                 MemoryBuffer layoutBin;
-                if (exportBinaryType(layoutBin, expr))
+                if (exportBinaryType(layoutBin, expr, isIndex))
                     return createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr);
             }
         }

ecl/eclcc/eclcc.cpp

@@ -2265,21 +2265,11 @@ IHqlExpression *EclCC::lookupDFSlayout(const char *filename, IErrorReceiver &err
                     else
                     {
                         diskRecord.set(diskRecord->queryBody());  // Remove location info - it's meaningless
-                        // MORE - if we already have the payload size info from the parsing of the ECL (we should, for all files built since 2017)
-                        // then don't parse _record_layout (which we expect to remove, eventually).
                         if (dfsFile->queryAttributes().hasProp("_record_layout"))
                         {
                             MemoryBuffer mb;
                             dfsFile->queryAttributes().getPropBin("_record_layout", mb);
-                            Owned<IDefRecordMeta> meta = deserializeRecordMeta(mb, true);
-                            int numKeyed = meta->numKeyedFields();
-                            if (numKeyed)
-                            {
-                                // NOTE - the index puts the payload on the no_newkeyindex, not on the record - so we will have to migrate it there
-                                int dfsPayload = getFlatFieldCount(diskRecord) - numKeyed;
-                                assertex(dfsPayload >= 0);
-                                diskRecord.setown(appendOwnedOperand(diskRecord, createAttribute(_payload_Atom, createConstant(dfsPayload))));
-                            }
+                            diskRecord.setown(patchEclRecordDefinitionFromRecordLayout(diskRecord, mb));
                         }
                     }
                 }

ecl/hql/hqlexpr.cpp

@@ -14312,27 +14312,60 @@ void exportMap(IPropertyTree *dataNode, IHqlExpression *destTable, IHqlExpressio
     maps->addPropTree("MapTables", map);
 }
 
-void exportJsonType(StringBuffer &ret, IHqlExpression *table)
+bool hasTrailingFilePos(IHqlExpression *record)
 {
-    Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
-    const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
-    dumpTypeInfo(ret, typeInfo);
+    unsigned numFields = record->numChildren();
+    if (numFields>1)
+    {
+        IHqlExpression * lastField = record->queryChild(numFields-1);
+        ITypeInfo * fileposType = lastField->queryType();
+        if (isSimpleIntegralType(fileposType))
+            return true;
+    }
+    return false;
 }
 
-bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table)
+void exportJsonType(StringBuffer &ret, IHqlExpression *table, bool forceIndex)
 {
-    try
+    if (forceIndex)
+    {
+        // When constructing from old index metadata, we don't know if FILEPOSITION(false) was specified on the index
+        // But we can have a reasonable guess - if no payload is specified, then there can't be a trailing fileposition field ...
+        OwnedHqlExpr indexRec = createMetadataIndexRecord(table, table->hasAttribute(_payload_Atom) && hasTrailingFilePos(table));
+        exportJsonType(ret, indexRec, false);
+    }
+    else
     {
         Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
         const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
-        return dumpTypeInfo(ret, typeInfo);
+        dumpTypeInfo(ret, typeInfo);
     }
-    catch (IException * e)
+}
+
+bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table, bool forceIndex)
+{
+    if (forceIndex)
     {
-        DBGLOG(e);
-        e->Release();
+        // When constructing from old index metadata, we don't know if FILEPOSITION(false) was specified on the index
+        // But we can have a reasonable guess - if no payload is specified, then there can't be a trailing fileposition field ...
+        OwnedHqlExpr indexRec = createMetadataIndexRecord(table, table->hasAttribute(_payload_Atom) && hasTrailingFilePos(table));
+        return exportBinaryType(ret, indexRec, false);
+    }
+    else
+    {
+        try
+        {
+            Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+            const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
+            return dumpTypeInfo(ret, typeInfo);
+        }
+        catch (IException * e)
+        {
+            DBGLOG(e);
+            e->Release();
+        }
+        return false;
     }
-    return false;
 }
 
 const RtlTypeInfo *queryRtlType(IRtlFieldTypeDeserializer &deserializer, IHqlExpression *table)

ecl/hql/hqlexpr.hpp

@@ -1900,9 +1900,10 @@ struct RtlTypeInfo;
 interface IRtlFieldTypeDeserializer;
 
 extern HQL_API void exportData(IPropertyTree *data, IHqlExpression *table, bool flatten=false);
-extern HQL_API void exportJsonType(StringBuffer &ret, IHqlExpression *table);
-extern HQL_API bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table);
+extern HQL_API void exportJsonType(StringBuffer &ret, IHqlExpression *table, bool forceIndex);
+extern HQL_API bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table, bool forceIndex);
 extern HQL_API const RtlTypeInfo *queryRtlType(IRtlFieldTypeDeserializer &deserializer, IHqlExpression *table);
+extern HQL_API bool hasTrailingFilePos(IHqlExpression *record);
 
 extern HQL_API void clearCacheCounts();
 extern HQL_API void displayHqlCacheStats();

ecl/hql/hqlgram2.cpp

@@ -7427,21 +7427,9 @@ void HqlGram::checkSoapRecord(attribute & errpos)
 
 IHqlExpression * HqlGram::checkIndexRecord(IHqlExpression * record, const attribute & errpos, OwnedHqlExpr & indexAttrs)
 {
-    unsigned numFields = record->numChildren();
-    if (numFields && getBoolAttributeInList(indexAttrs, filepositionAtom, true))
+    if (record->numChildren() && getBoolAttributeInList(indexAttrs, filepositionAtom, true) && !hasTrailingFilePos(record))
     {
-        // if not, implies some error (already reported)
-        if (numFields == 1)
-        {
-            indexAttrs.setown(createComma(indexAttrs.getClear(), createExprAttribute(filepositionAtom, createConstant(false))));
-        }
-        else
-        {
-            IHqlExpression * lastField = record->queryChild(numFields-1);
-            ITypeInfo * fileposType = lastField->queryType();
-            if (!isSimpleIntegralType(fileposType))
-                indexAttrs.setown(createComma(indexAttrs.getClear(), createExprAttribute(filepositionAtom, createConstant(false))));
-        }
+        indexAttrs.setown(createComma(indexAttrs.getClear(), createExprAttribute(filepositionAtom, createConstant(false))));
     }
     return LINK(record);
 }

ecl/hql/hqlutil.cpp

@@ -347,6 +347,162 @@ IHqlExpression * getHozedKeyValue(IHqlExpression * _value)
     return ensureExprType(value, hozedType);
 }
 
+IHqlExpression * convertIndexPhysical2LogicalValue(IHqlExpression * cur, IHqlExpression * physicalSelect, bool allowTranslate)
+{
+    if (cur->hasAttribute(blobAtom))
+    {
+        if (cur->isDataset())
+            return createDataset(no_id2blob, LINK(physicalSelect), LINK(cur->queryRecord()));
+        else if (cur->isDatarow())
+            return createRow(no_id2blob, LINK(physicalSelect), LINK(cur->queryRecord()));
+        else
+            return createValue(no_id2blob, cur->getType(), LINK(physicalSelect));
+    }
+    else if (allowTranslate)
+    {
+        LinkedHqlExpr newValue = physicalSelect;
+
+        OwnedHqlExpr target = createSelectExpr(getActiveTableSelector(), LINK(cur));            // select not used, just created to get correct types.
+        ITypeInfo * type = target->queryType();
+        type_t tc = type->getTypeCode();
+        if (tc == type_int || tc == type_swapint)
+        {
+            if (type->isSigned())
+            {
+                Owned<ITypeInfo> tempType = makeIntType(type->getSize(), false);
+                newValue.setown(ensureExprType(newValue, tempType));
+                newValue.setown(createValue(no_sub, newValue->getType(), LINK(newValue), getHozedBias(newValue->queryType())));
+            }
+        }
+
+        return ensureExprType(newValue, type);
+    }
+    else
+        return ensureExprType(physicalSelect, cur->queryType());
+}
+
+static IHqlExpression * mapIfBlock(HqlMapTransformer & mapper, IHqlExpression * cur);
+static IHqlExpression * mapIfBlockRecord(HqlMapTransformer & mapper, IHqlExpression * record)
+{
+    HqlExprArray mapped;
+    ForEachChild(i, record)
+    {
+        IHqlExpression * cur = record->queryChild(i);
+        if (cur->getOperator() == no_ifblock)
+            mapped.append(*mapIfBlock(mapper, cur));
+        else
+            mapped.append(*LINK(cur));
+    }
+    return record->clone(mapped);
+}
+
+
+static IHqlExpression * mapIfBlock(HqlMapTransformer & mapper, IHqlExpression * cur)
+{
+    HqlExprArray args;
+    unwindChildren(args, cur);
+    args.replace(*mapper.transformRoot(&args.item(0)), 0);
+    args.replace(*mapIfBlockRecord(mapper, &args.item(1)), 1);
+    return cur->clone(args);
+}
+
+
+IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpression * record, bool hasInternalFileposition, bool createKeyedTypes)
+{
+    HqlExprArray physicalFields;
+    unsigned max = record->numChildren() - (hasInternalFileposition ? 1 : 0);
+    for (unsigned idx=0; idx < max; idx++)
+    {
+        IHqlExpression * cur = record->queryChild(idx);
+        IHqlExpression * newField = NULL;
+
+        if (cur->isAttribute())
+            physicalFields.append(*LINK(cur));
+        else if (cur->getOperator() == no_ifblock)
+            physicalFields.append(*mapIfBlock(mapper, cur));
+        else if (cur->getOperator() == no_record)
+            physicalFields.append(*createPhysicalIndexRecord(mapper, cur, false, createKeyedTypes));
+        else if (cur->hasAttribute(blobAtom))
+        {
+            newField = createField(cur->queryId(), makeIntType(8, false), nullptr, createKeyedTypes ? createAttribute(_payload_Atom) : nullptr);
+        }
+        else
+        {
+            //This should support other non serialized formats.  E.g., link counted strings.
+            //Simplest would be to move getSerializedForm code + call that first.
+            if (cur->hasAttribute(_linkCounted_Atom) || cur->isDatarow())
+            {
+                newField = getSerializedForm(cur, diskAtom);
+                assertex(newField != cur || cur->isDatarow());
+            }
+            else
+            {
+                Owned<ITypeInfo> hozedType = getHozedKeyType(cur);
+                if (hozedType == cur->queryType())
+                    newField = LINK(cur);
+                else if (createKeyedTypes)
+                    newField = createField(cur->queryId(), makeKeyedType(cur->getType()), nullptr, extractFieldAttrs(cur));
+                else
+                    newField = createField(cur->queryId(), hozedType.getClear(), nullptr, extractFieldAttrs(cur));
+            }
+        }
+
+        if (newField)
+        {
+            physicalFields.append(*newField);
+            if (cur != newField)
+            {
+                IHqlExpression * self = querySelfReference();
+                OwnedHqlExpr select = createSelectExpr(LINK(self), LINK(cur));
+                OwnedHqlExpr physicalSelect = createSelectExpr(LINK(self), LINK(newField));
+                OwnedHqlExpr newValue = convertIndexPhysical2LogicalValue(cur, physicalSelect, true);
+                mapper.setMapping(select, newValue);
+            }
+        }
+    }
+    if (hasInternalFileposition)
+    {
+        if (createKeyedTypes)
+        {
+            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
+            IHqlExpression *fposField = createField(cur->queryId(), makeFilePosType(cur->getType()), nullptr, extractFieldAttrs(cur));
+            physicalFields.append(*fposField);
+        }
+        else
+        {
+            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
+            IHqlExpression *fposField = createField(cur->queryId(), makeSwapIntType(8, false), nullptr, extractFieldAttrs(cur));
+            physicalFields.append(*fposField);
+        }
+    }
+
+    return createRecord(physicalFields);
+}
+
+IHqlExpression * createMetadataIndexRecord(IHqlExpression * record, bool hasInternalFilePosition)
+{
+    HqlMapTransformer mapper;
+    return createPhysicalIndexRecord(mapper, record, hasInternalFilePosition, true);
+}
+
+extern HQL_API IHqlExpression * patchEclRecordDefinitionFromRecordLayout(IHqlExpression * record, MemoryBuffer &mb)
+{
+    if (!record->hasAttribute(_payload_Atom))
+    {
+        Owned<IDefRecordMeta> meta = deserializeRecordMeta(mb, true);
+        int numKeyed = meta->numKeyedFields();
+        if (numKeyed)
+        {
+            // NOTE - the index puts the payload on the no_newkeyindex, not on the record - so we will have to migrate it there
+            int dfsPayload = getFlatFieldCount(record) - numKeyed;
+            assertex(dfsPayload >= 0);
+            if (dfsPayload)
+                return prependOwnedOperand(record, createAttribute(_payload_Atom, createConstant(dfsPayload)));
+        }
+    }
+    return LINK(record);
+}
+
 //---------------------------------------------------------------------------
 
 bool containsAggregate(IHqlExpression * expr)
@@ -5199,6 +5355,16 @@ IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlExpression * owne
     return expr->clone(args);
 }
 
+IHqlExpression * prependOwnedOperand(IHqlExpression * expr, IHqlExpression * ownedOperand)
+{
+    if (!ownedOperand)
+        return LINK(expr);
+    HqlExprArray args;
+    args.append(*ownedOperand);
+    unwindChildren(args, expr);
+    return expr->clone(args);
+}
+
 IHqlExpression * removeOperand(IHqlExpression * expr, IHqlExpression * operand)
 {
     HqlExprArray args;

ecl/hql/hqlutil.hpp

@@ -202,6 +202,7 @@ extern HQL_API IHqlExpression * removeChildOp(IHqlExpression * expr, node_operat
 extern HQL_API IHqlExpression * removeChild(IHqlExpression * expr, unsigned child);
 extern HQL_API IHqlExpression * appendAttribute(IHqlExpression * expr, IAtom * attr);
 extern HQL_API IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlExpression * ownedOperand);
+extern HQL_API IHqlExpression * prependOwnedOperand(IHqlExpression * expr, IHqlExpression * ownedOperand);
 extern HQL_API IHqlExpression * replaceOwnedAttribute(IHqlExpression * expr, IHqlExpression * ownedAttribute);
 extern HQL_API IHqlExpression * appendOwnedOperandsF(IHqlExpression * expr, ...);
 extern HQL_API IHqlExpression * inheritAttribute(IHqlExpression * expr, IHqlExpression * donor, IAtom * name);
@@ -690,6 +691,12 @@ extern HQL_API IHqlExpression * getHozedBias(ITypeInfo * type);
 extern HQL_API ITypeInfo * getHozedKeyType(IHqlExpression * expr);
 extern HQL_API IHqlExpression * getHozedKeyValue(IHqlExpression * _value);
 
+class HqlMapTransformer;
+extern HQL_API IHqlExpression * convertIndexPhysical2LogicalValue(IHqlExpression * cur, IHqlExpression * physicalSelect, bool allowTranslate);
+extern HQL_API IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpression * record, bool hasInternalFileposition, bool createKeyedTypes);
+extern HQL_API IHqlExpression * createMetadataIndexRecord(IHqlExpression * record, bool hasInternalFilePosition);
+extern HQL_API IHqlExpression * patchEclRecordDefinitionFromRecordLayout(IHqlExpression * record, MemoryBuffer &recordLayoutInfo);
+
 extern HQL_API bool hasNonNullRecord(ITypeInfo * type);
 
 //Mangle the names to make it slightly trickier for someone disassembling the system, 

ecl/hqlcpp/hqlckey.cpp

@@ -103,41 +103,6 @@ bool isKeyableType(ITypeInfo * type)
     }
 }
 
-IHqlExpression * convertIndexPhysical2LogicalValue(IHqlExpression * cur, IHqlExpression * physicalSelect, bool allowTranslate)
-{
-    if (cur->hasAttribute(blobAtom))
-    {
-        if (cur->isDataset())
-            return createDataset(no_id2blob, LINK(physicalSelect), LINK(cur->queryRecord()));
-        else if (cur->isDatarow())
-            return createRow(no_id2blob, LINK(physicalSelect), LINK(cur->queryRecord()));
-        else
-            return createValue(no_id2blob, cur->getType(), LINK(physicalSelect));
-    }
-    else if (allowTranslate)
-    {
-        LinkedHqlExpr newValue = physicalSelect;
-
-        OwnedHqlExpr target = createSelectExpr(getActiveTableSelector(), LINK(cur));            // select not used, just created to get correct types.
-        ITypeInfo * type = target->queryType();
-        type_t tc = type->getTypeCode();
-        if (tc == type_int || tc == type_swapint)
-        {
-            if (type->isSigned())
-            {
-                Owned<ITypeInfo> tempType = makeIntType(type->getSize(), false);
-                newValue.setown(ensureExprType(newValue, tempType));
-                newValue.setown(createValue(no_sub, newValue->getType(), LINK(newValue), getHozedBias(newValue->queryType())));
-            }
-        }
-
-        return ensureExprType(newValue, type);
-    }
-    else
-        return ensureExprType(physicalSelect, cur->queryType());
-}
-
-
 //--------------------------------------------------------------------------------------------------
 
 void HqlCppTranslator::buildJoinMatchFunction(BuildCtx & ctx, const char * name, IHqlExpression * left, IHqlExpression * right, IHqlExpression * match, IHqlExpression * selSeq)

ecl/hqlcpp/hqlhtcpp.ipp

@@ -243,7 +243,6 @@ public:
 
 
 unsigned getVirtualFieldSize(IHqlExpression * record);
-IHqlExpression * convertIndexPhysical2LogicalValue(IHqlExpression * cur, IHqlExpression * physicalSelect, bool allowTranslate);
 bool requiresHozedTransform(ITypeInfo * type);
 bool isKeyableType(ITypeInfo * type);
 IHqlExpression * getFilepos(IHqlExpression * dataset, bool isLocal);

ecl/hqlcpp/hqlsource.cpp

@@ -462,111 +462,6 @@ static void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression
 }
 
 
-static IHqlExpression * mapIfBlock(HqlMapTransformer & mapper, IHqlExpression * cur);
-static IHqlExpression * mapIfBlockRecord(HqlMapTransformer & mapper, IHqlExpression * record)
-{
-    HqlExprArray mapped;
-    ForEachChild(i, record)
-    {
-        IHqlExpression * cur = record->queryChild(i);
-        if (cur->getOperator() == no_ifblock)
-            mapped.append(*mapIfBlock(mapper, cur));
-        else
-            mapped.append(*LINK(cur));
-    }
-    return record->clone(mapped);
-}
-
-
-static IHqlExpression * mapIfBlock(HqlMapTransformer & mapper, IHqlExpression * cur)
-{
-    HqlExprArray args;
-    unwindChildren(args, cur);
-    args.replace(*mapper.transformRoot(&args.item(0)), 0);
-    args.replace(*mapIfBlockRecord(mapper, &args.item(1)), 1);
-    return cur->clone(args);
-}
-
-
-static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpression * record, bool hasInternalFileposition, bool createKeyedTypes)
-{
-    HqlExprArray physicalFields;
-    unsigned max = record->numChildren() - (hasInternalFileposition ? 1 : 0);
-    for (unsigned idx=0; idx < max; idx++)
-    {
-        IHqlExpression * cur = record->queryChild(idx);
-        IHqlExpression * newField = NULL;
-
-        if (cur->isAttribute())
-            physicalFields.append(*LINK(cur));
-        else if (cur->getOperator() == no_ifblock)
-            physicalFields.append(*mapIfBlock(mapper, cur));
-        else if (cur->getOperator() == no_record)
-            physicalFields.append(*createPhysicalIndexRecord(mapper, cur, false, createKeyedTypes));
-        else if (cur->hasAttribute(blobAtom))
-        {
-            newField = createField(cur->queryId(), makeIntType(8, false), nullptr, createKeyedTypes ? createAttribute(_payload_Atom) : nullptr);
-        }
-        else
-        {
-            //This should support other non serialized formats.  E.g., link counted strings.
-            //Simplest would be to move getSerializedForm code + call that first.
-            if (cur->hasAttribute(_linkCounted_Atom) || cur->isDatarow())
-            {
-                newField = getSerializedForm(cur, diskAtom);
-                assertex(newField != cur || cur->isDatarow());
-            }
-            else
-            {
-                Owned<ITypeInfo> hozedType = getHozedKeyType(cur);
-                if (hozedType == cur->queryType())
-                    newField = LINK(cur);
-                else if (createKeyedTypes)
-                    newField = createField(cur->queryId(), makeKeyedType(cur->getType()), nullptr, extractFieldAttrs(cur));
-                else
-                    newField = createField(cur->queryId(), hozedType.getClear(), nullptr, extractFieldAttrs(cur));
-            }
-        }
-
-        if (newField)
-        {
-            physicalFields.append(*newField);
-            if (cur != newField)
-            {
-                IHqlExpression * self = querySelfReference();
-                OwnedHqlExpr select = createSelectExpr(LINK(self), LINK(cur));
-                OwnedHqlExpr physicalSelect = createSelectExpr(LINK(self), LINK(newField));
-                OwnedHqlExpr newValue = convertIndexPhysical2LogicalValue(cur, physicalSelect, true);
-                mapper.setMapping(select, newValue);
-            }
-        }
-    }
-    if (hasInternalFileposition)
-    {
-        if (createKeyedTypes)
-        {
-            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
-            IHqlExpression *fposField = createField(cur->queryId(), makeFilePosType(cur->getType()), nullptr, extractFieldAttrs(cur));
-            physicalFields.append(*fposField);
-        }
-        else
-        {
-            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
-            IHqlExpression *fposField = createField(cur->queryId(), makeSwapIntType(8, false), nullptr, extractFieldAttrs(cur));
-            physicalFields.append(*fposField);
-        }
-    }
-
-    return createRecord(physicalFields);
-}
-
-IHqlExpression * createMetadataIndexRecord(IHqlExpression * record, bool hasInternalFilePosition)
-{
-    HqlMapTransformer mapper;
-    return createPhysicalIndexRecord(mapper, record, hasInternalFilePosition, true);
-}
-
-
 IHqlExpression * HqlCppTranslator::convertToPhysicalIndex(IHqlExpression * tableExpr)
 {
     LinkedHqlExpr * match = physicalIndexCache.getValue(tableExpr);

esp/services/ws_dfu/ws_dfuService.cpp

@@ -53,6 +53,7 @@
 
 #include "hqlerror.hpp"
 #include "hqlexpr.hpp"
+#include "hqlutil.hpp"
 #include "eclrtl.hpp"
 #include "package.h"
 #include "daaudit.hpp"
@@ -1653,20 +1654,24 @@ bool CWsDfuEx::onDFURecordTypeInfo(IEspContext &context, IEspDFURecordTypeInfoRe
         else if (df->queryAttributes().hasProp("ECL"))
         {
             const char * kind = df->queryAttributes().queryProp("@kind");
-            if (kind && streq(kind, "key"))
-                throw MakeStringException(ECLWATCH_FILE_NOT_EXIST, "Index file %s does not contain type information",fileName);
-
+            bool isIndex = (kind && streq(kind, "key"));
             OwnedHqlExpr record = getEclRecordDefinition(userdesc, fileName);
+            if (df->queryAttributes().hasProp("_record_layout"))
+            {
+                MemoryBuffer mb;
+                df->queryAttributes().getPropBin("_record_layout", mb);
+                record.setown(patchEclRecordDefinitionFromRecordLayout(record, mb));
+            }
             if (req.getIncludeJsonTypeInfo())
             {
                 StringBuffer jsonFormat;
-                exportJsonType(jsonFormat,record);
+                exportJsonType(jsonFormat, record, isIndex);
                 resp.setJsonInfo(jsonFormat);
             }
             if (req.getIncludeBinTypeInfo())
             {
                 MemoryBuffer binFormat;
-                exportBinaryType(binFormat,record);
+                exportBinaryType(binFormat, record, isIndex);
                 resp.setBinInfo(binFormat);
             }
         }
@@ -1686,13 +1691,13 @@ bool CWsDfuEx::onEclRecordTypeInfo(IEspContext &context, IEspEclRecordTypeInfoRe
         if (req.getIncludeJsonTypeInfo())
         {
             StringBuffer jsonFormat;
-            exportJsonType(jsonFormat,record);
+            exportJsonType(jsonFormat, record, false);  // MORE - could allow isIndex to be passed in?
             resp.setJsonInfo(jsonFormat);
         }
         if (req.getIncludeBinTypeInfo())
         {
             MemoryBuffer binFormat;
-            exportBinaryType(binFormat,record);
+            exportBinaryType(binFormat, record, false);
             resp.setBinInfo(binFormat);
         }
     }
@@ -2388,19 +2393,24 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
         else if (df->queryAttributes().hasProp("ECL"))
         {
             const char * kind = df->queryAttributes().queryProp("@kind");
-            if (kind && streq(kind, "key"))
-                throw MakeStringException(ECLWATCH_FILE_NOT_EXIST, "Index file %s does not contain type information", name);
+            bool isIndex = (kind && streq(kind, "key"));
             OwnedHqlExpr record = getEclRecordDefinition(df->queryAttributes().queryProp("ECL"));
+            if (df->queryAttributes().hasProp("_record_layout"))
+            {
+                MemoryBuffer mb;
+                df->queryAttributes().getPropBin("_record_layout", mb);
+                record.setown(patchEclRecordDefinitionFromRecordLayout(record, mb));
+            }
             if (includeJsonTypeInfo)
             {
                 StringBuffer jsonFormat;
-                exportJsonType(jsonFormat, record);
+                exportJsonType(jsonFormat, record, isIndex);
                 FileDetails.setJsonInfo(jsonFormat);
             }
             if (includeBinTypeInfo)
             {
                 MemoryBuffer binFormat;
-                exportBinaryType(binFormat, record);
+                exportBinaryType(binFormat, record, isIndex);
                 FileDetails.setBinInfo(binFormat);
             }
         }