Merge pull request #10683 from richardkchapman/dumpkey

HPCC-18796 Recent filepos changes have broken dumpkey

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/hql/hqlexpr.cpp

@@ -14071,7 +14071,7 @@ void exportBinaryType(MemoryBuffer &ret, IHqlExpression *table)
 {
     Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
     const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
-    dumpTypeInfo(ret, typeInfo, false);
+    dumpTypeInfo(ret, typeInfo);
 }
 
 const RtlTypeInfo *queryRtlType(IRtlFieldTypeDeserializer &deserializer, IHqlExpression *table)

ecl/hql/hqlutil.cpp

@@ -9687,7 +9687,8 @@ void getFieldTypeInfo(FieldTypeInfoStruct &out, ITypeInfo *type)
     case type_filepos:
         out.className = "RtlSwapIntTypeInfo";
         out.length = sizeof(offset_t);
-        out.fieldType |= RFTMunsigned;
+        if (!type->isSigned())
+            out.fieldType |= RFTMunsigned;
         break;
     case type_blob:
         out.className = "RtlBlobTypeInfo";

ecl/hqlcpp/hqlsource.cpp

@@ -500,7 +500,7 @@ static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IH
                 if (hozedType == cur->queryType())
                     newField = LINK(cur);
                 else if (createKeyedTypes)
-                    newField = createField(cur->queryId(), makeKeyedType(hozedType.getClear()), nullptr, extractFieldAttrs(cur));
+                    newField = createField(cur->queryId(), makeKeyedType(cur->getType()), nullptr, extractFieldAttrs(cur));
                 else
                     newField = createField(cur->queryId(), hozedType.getClear(), nullptr, extractFieldAttrs(cur));
             }

ecl/hthor/hthor.cpp

@@ -1214,10 +1214,10 @@ void CHThorIndexWriteActivity::execute()
         rtlFree(layoutMetaBuff);
     }
     // New record layout info
-    if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
+    if (helper.queryDiskRecordSize()->queryTypeInfo())
     {
         MemoryBuffer out;
-        dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
         properties.setPropBin("_rtlType", out.length(), out.toByteArray());
     }
 
@@ -1284,10 +1284,10 @@ void CHThorIndexWriteActivity::buildLayoutMetadata(Owned<IPropertyTree> & metada
     if(!metadata) metadata.setown(createPTree("metadata"));
     metadata->setProp("_record_ECL", helper.queryRecordECL());
 
-    if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
+    if (helper.queryDiskRecordSize()->queryTypeInfo())
     {
         MemoryBuffer out;
-        dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
         metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
     }
 }

roxie/ccd/ccdserver.cpp

@@ -12014,10 +12014,10 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
             metadata.setown(createPTree("metadata", ipt_fast));
         metadata->setProp("_record_ECL", helper.queryRecordECL());
 
-        if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
+        if (helper.queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+            dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
             metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }
@@ -12224,10 +12224,10 @@ public:
             rtlFree(layoutMetaBuff);
         }
         // New record layout info
-        if (helper.queryOutputMeta() && helper.queryOutputMeta()->queryTypeInfo())
+        if (helper.queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper.queryOutputMeta()->queryTypeInfo(), true);
+            dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
             properties.setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }

rtl/eclrtl/eclhelper_dyn.cpp

@@ -151,14 +151,14 @@ public:
             {
                 filters.append(nullptr);
                 filterOffsets.append(offsetCalculator.getOffset(filters.length()));
-                filterOffsets.append(offsetCalculator.getSize(filters.length()));
+                filterSizes.append(offsetCalculator.getSize(filters.length()));
             }
             IStringSet *prev = filters.item(fieldNum);
             if (prev)
                 filterSet.setown(prev->unionSet(filterSet)); // Debatable - would intersect be more appropriate?
             filters.replace(filterSet.getClear(), fieldNum);
-            filterOffsets.replace(fieldOffset, fieldOffset); // MORE - probably refactor this in  a bit
-            filterSizes.replace(fieldSize, fieldSize); // MORE - probably refactor this in  a bit
+            filterOffsets.replace(fieldOffset, fieldNum); // MORE - probably refactor this in  a bit
+            filterSizes.replace(fieldSize, fieldNum); // MORE - probably refactor this in  a bit
         }
     }
     void createSegmentMonitors(IIndexReadContext *irc)

rtl/eclrtl/rtldynfield.cpp

@@ -317,132 +317,6 @@ private:
     bool commaPending = false;
 };
 
-class IndexBiasTranslator
-{
-public:
-    IndexBiasTranslator(const RtlTypeInfo *type)
-    {
-        translatedType = type;  // Assume no translation needed until proven otherwise
-        if (type->getType() != type_record)
-            return;
-        const RtlFieldInfo * const * fields = type->queryFields();
-        if (!fields)
-            return;
-
-        unsigned numFields;
-        needsTranslation = false;
-        for (numFields=0;;numFields++)
-        {
-            const RtlFieldInfo * child = fields[numFields];
-            if (!child)
-                break;
-            switch (child->type->getType())
-            {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-            case type_swapint:
-                if (!child->type->isUnsigned())
-                    needsTranslation = true;
-                break;
-            case type_int:
-                needsTranslation = true;
-                break;
-#else
-            case type_int:
-                if (!child->type->isUnsigned())
-                    needsTranlsation = true;
-                break;
-            case type_swapint:
-                needsTranslation = true;
-                break;
-#endif
-            }
-        }
-        if (!needsTranslation && numFields > 1)
-        {
-            // Check if need last field translating to a type_filepos
-            switch(fields[numFields]->type->getType())
-            {
-            case type_int:
-            case type_swapint:
-            case type_packedint:
-            case type_bitfield:
-                needsTranslation = true;
-                break;
-            }
-        }
-        if (needsTranslation)
-        {
-            translated = new bool[numFields];
-            RtlFieldInfo * * newFields = new RtlFieldInfo * [numFields+1];
-            newFields[numFields] = nullptr;
-            for (unsigned idx = 0; idx < numFields; idx++)
-            {
-                newFields[idx] = new RtlFieldInfo(*fields[idx]);
-                const RtlTypeInfo *newType = createBiasType(fields[idx]->type, idx > 1 && idx == numFields-1);
-                // MORE - Is it an issue if we don't common these up?
-                if (newType)
-                {
-                    newFields[idx]->type = newType;
-                    translated[idx] = true;
-                }
-                else
-                    translated[idx] = false;
-            }
-            translatedType = new RtlRecordTypeInfo(type->fieldType, type->length, newFields);
-        }
-    }
-    ~IndexBiasTranslator()
-    {
-        if (needsTranslation)
-        {
-            const RtlFieldInfo * const * fields = translatedType->queryFields();
-            for (unsigned idx = 0;;idx++)
-            {
-                const RtlFieldInfo * child = fields[idx];
-                if (!child)
-                    break;
-                if (translated[idx])
-                    child->type->doDelete();
-                delete child;
-            }
-            delete [] fields;
-            translatedType->doDelete();
-            delete [] translated;
-        }
-    }
-    const RtlTypeInfo *queryTranslatedType()
-    {
-        return translatedType;
-    }
-private:
-    static const RtlTypeInfo *createBiasType(const RtlTypeInfo *origType, bool isLastField)
-    {
-        auto type = origType->getType();
-        if (type==type_int || type==type_swapint)
-        {
-            unsigned flags = origType->fieldType & ~RFTMkind;
-            unsigned length = origType->length;
-            if (isLastField)
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-                return new RtlSwapIntTypeInfo(type_filepos|type_unsigned, sizeof(offset_t));
-#else
-                return new RtlIntTypeInfo(type_filepos|type_unsigned, sizeof(offset_t));
-#endif
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-            else if (type == type_int || origType->isSigned())
-#else
-            else if (type == type_swapint || origType->isSigned()) // MORE - this may not be right if compiler machine endianness does not match this machine
-#endif
-                return new RtlKeyedIntTypeInfo(type_keyedint | flags, length, origType);
-        }
-        return nullptr;
-    }
-
-    const RtlTypeInfo *translatedType = nullptr;
-    bool needsTranslation = false;
-    bool *translated = nullptr;
-};
-
 class CRtlFieldTypeBinSerializer
 {
 public:
@@ -453,7 +327,7 @@ public:
      * @param  type RtlTypeInfo structure to be serialized
      * @return Referenced to supplied buffer
      */
-     static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type, bool applyBias)
+     static MemoryBuffer &serialize(MemoryBuffer &out, const RtlTypeInfo *type)
      {
          int oldEnd = out.setEndian(__LITTLE_ENDIAN);
          CRtlFieldTypeBinSerializer s(out);
@@ -462,13 +336,7 @@ public:
          DelayedMarker<hash64_t> hash(out);
          DelayedSizeMarker size(out);
          size32_t pos = out.length();
-         if (applyBias)
-         {
-             IndexBiasTranslator translator(type);
-             s.serializeType(translator.queryTranslatedType());
-         }
-         else
-             s.serializeType(type);
+         s.serializeType(type);
          size.write();
          hash.write(rtlHash64Data(size.size(), out.toByteArray()+pos, 0));
          out.setEndian(oldEnd);
@@ -907,15 +775,15 @@ extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInf
     return CRtlFieldTypeSerializer::serialize(ret, t);
 }
 
-extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t, bool useBias)
+extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
 {
-    return CRtlFieldTypeBinSerializer::serialize(ret, t, useBias);
+    return CRtlFieldTypeBinSerializer::serialize(ret, t);
 }
 
 extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __result, IOutputMetaData &  metaVal)
 {
     MemoryBuffer ret;
-    CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo(), false);
+    CRtlFieldTypeBinSerializer::serialize(ret, metaVal.queryTypeInfo());
     __lenResult = ret.length();
     __result = ret.detach();
 }
@@ -931,7 +799,7 @@ extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,I
     CRtlFieldTypeSerializer::serialize(ret2, deserializer.deserialize(ret));
     assert(streq(ret, ret2));
     MemoryBuffer out;
-    CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo(), false);
+    CRtlFieldTypeBinSerializer::serialize(out, metaVal.queryTypeInfo());
     CRtlFieldTypeDeserializer bindeserializer(nullptr);
     CRtlFieldTypeSerializer::serialize(ret2.clear(), bindeserializer.deserialize(out));
     assert(streq(ret, ret2));

rtl/eclrtl/rtldynfield.hpp

@@ -115,7 +115,7 @@ extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(ITho
 
 extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t);
 
-extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t, bool useBias);
+extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t);
 
 /**
  * Serialize metadata of supplied record to JSON, and return it to ECL caller as a string. Used for testing serializer.

rtl/eclrtl/rtlfield.cpp

@@ -2732,7 +2732,7 @@ extern bool ECLRTL_API hasTrailingFileposition(const RtlFieldInfo * const * fiel
         return false;
     while (*fields)
         fields++;
-    return (fields[-1]->type->getType() == type_filepos);
+    return (fields[-1]->type->getType() == type_filepos);  // note - filepos is created as an RtlSwapIntTypeInfo with type set to type_filepos
 }
 
 extern bool ECLRTL_API hasTrailingFileposition(const RtlTypeInfo * type)

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -178,10 +178,10 @@ public:
             rtlFree(layoutMetaBuff);
         }
         // New record layout info
-        if (helper->queryOutputMeta() && helper->queryOutputMeta()->queryTypeInfo())
+        if (helper->queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper->queryOutputMeta()->queryTypeInfo(), true);
+            dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo());
             props.setPropBin("_rtlType", out.length(), out.toByteArray());
         }
         mpTag = container.queryJob().allocateMPTag();

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -204,10 +204,10 @@ public:
         if(!metadata) metadata.setown(createPTree("metadata"));
         metadata->setProp("_record_ECL", helper->queryRecordECL());
 
-        if (helper->queryOutputMeta() && helper->queryOutputMeta()->queryTypeInfo())
+        if (helper->queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper->queryOutputMeta()->queryTypeInfo(), true);
+            dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo());
             metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }