HPCC-8551 Fix additional issues with serialization

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlattr.cpp

@@ -3269,6 +3269,27 @@ bool recordSerializationDiffers(IHqlExpression * expr, _ATOM serializeForm1, _AT
     return querySerializedForm(expr, serializeForm1) != querySerializedForm(expr, serializeForm2);
 }
 
+extern HQL_API bool typeRequiresDeserialization(ITypeInfo * type, _ATOM serializeForm)
+{
+    Owned<ITypeInfo> serializedType = getSerializedForm(type, serializeForm);
+
+    if (queryUnqualifiedType(serializedType) == queryUnqualifiedType(type))
+        return false;
+
+    type_t stc = serializedType->getTypeCode();
+    if (stc != type->getTypeCode())
+        return true;
+
+    if (stc == type_table)
+    {
+        if (recordTypesMatch(serializedType, type))
+            return false;
+        return true;
+    }
+
+    return true;
+}
+
 //---------------------------------------------------------------------------------
 
 IHqlExpression * queryRecordCountInfo(IHqlExpression * expr)

ecl/hql/hqlattr.hpp

@@ -43,6 +43,7 @@ extern HQL_API bool recordRequiresLinkCount(IHqlExpression * expr);
 extern HQL_API bool recordRequiresDestructor(IHqlExpression * expr);
 //extern HQL_API bool recordRequiresSerialization(IHqlExpression * expr);
 extern HQL_API bool recordRequiresSerialization(IHqlExpression * expr, _ATOM serializeForm);
+extern HQL_API bool typeRequiresDeserialization(ITypeInfo * type, _ATOM serializeForm); // or can we use the serialized form directly
 extern HQL_API bool recordSerializationDiffers(IHqlExpression * expr, _ATOM serializeForm1, _ATOM serializeForm2);
 extern HQL_API IHqlExpression * getSerializedForm(IHqlExpression * expr, _ATOM variation);
 extern HQL_API ITypeInfo * getSerializedForm(ITypeInfo * type, _ATOM variation);

ecl/hql/hqlexpr.cpp

@@ -12482,8 +12482,9 @@ IHqlExpression * ensureSerialized(IHqlExpression * expr, _ATOM serialForm)
 
 IHqlExpression * ensureDeserialized(IHqlExpression * expr, ITypeInfo * type, _ATOM serialForm)
 {
+    assertex(type->getTypeCode() != type_record);
     Owned<ITypeInfo> serialType = getSerializedForm(type, serialForm);
-    if (type == serialType)
+    if (queryUnqualifiedType(type) == queryUnqualifiedType(serialType))
         return LINK(expr);
 
     assertRecordTypesMatch(expr->queryType(), serialType);

ecl/hql/hqlgram2.cpp

@@ -1480,7 +1480,8 @@ void HqlGram::doAddAssignment(IHqlExpression * transform, IHqlExpression * _fiel
         getFriendlyTypeStr(rhsType,msg).append(" to ");
         getFriendlyTypeStr(fldType,msg).append(" (field ");
         getFldName(field,msg).append(")");
-        reportError(ERR_TYPE_INCOMPATIBLE,errpos, "%s", msg.str()); 
+        reportError(ERR_TYPE_INCOMPATIBLE,errpos, "%s", msg.str());
+        rhs.setown(createNullExpr(field));
     }
 
     appendTransformAssign(transform, field, rhs, errpos);

ecl/hqlcpp/hqlckey.cpp

@@ -644,7 +644,7 @@ void KeyedJoinInfo::buildTransformBody(BuildCtx & ctx, IHqlExpression * transfor
             //References to ROWS(originalRight) need to be replaced with DESERIALIZE(ROWS(serializedRight))
             OwnedHqlExpr originalRows = createDataset(no_rows, LINK(originalRight), LINK(rowsid));
             OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(extractedRight), LINK(rowsid));
-            OwnedHqlExpr deserializedRows = ensureDeserialized(rowsExpr, rhsRecord->queryType(), diskAtom);
+            OwnedHqlExpr deserializedRows = ensureDeserialized(rowsExpr, rhs->queryType(), diskAtom);
             newTransform.setown(replaceExpression(newTransform, originalRows, deserializedRows));
         }
     }

ecl/hqlcpp/hqlcppds.cpp

@@ -1899,16 +1899,18 @@ void HqlCppTranslator::buildAssignDeserializedDataset(BuildCtx & ctx, const CHql
 
 void HqlCppTranslator::buildDeserializedDataset(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * expr, CHqlBoundExpr & tgt, _ATOM serializeForm)
 {
+#ifdef _DEBUG
+    OwnedITypeInfo serializedType = getSerializedForm(type, serializeForm);
+    assertex(recordTypesMatch(expr->queryType(), serializedType));
+#endif
+
+    ITypeInfo * const exprType = expr->queryType();
+    assertex(!hasLinkedRow(exprType));
+
     CHqlBoundTarget target;
     createTempFor(ctx, type, target, typemod_none, FormatLinkedDataset);
 
-    if (hasLinkedRow(expr->queryType()))
-    {
-        throwUnexpected();
-        buildDatasetAssign(ctx, target, expr); // ???
-    }
-    else
-        buildAssignDeserializedDataset(ctx, target, expr, serializeForm);
+    buildAssignDeserializedDataset(ctx, target, expr, serializeForm);
 
     tgt.setFromTarget(target);
 }
@@ -2104,17 +2106,31 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             return;
         }
     case no_serialize:
-        if (isDummySerializeDeserialize(expr))
-            doBuildDataset(ctx, expr->queryChild(0)->queryChild(0), tgt, format);
-        else
-            buildSerializedDataset(ctx, expr->queryChild(0), tgt, expr->queryChild(1)->queryName());
-        return;
+        {
+            IHqlExpression * deserialized = expr->queryChild(0);
+            _ATOM serializeForm = expr->queryChild(1)->queryName();
+            if (isDummySerializeDeserialize(expr))
+                doBuildDataset(ctx, deserialized->queryChild(0), tgt, format);
+            else if (!typeRequiresDeserialization(deserialized->queryType(), serializeForm))
+                //Optimize creating a serialized version of a dataset if the record is the same serialized and unserialized
+                buildDataset(ctx, deserialized, tgt, FormatNatural);
+            else
+                buildSerializedDataset(ctx, deserialized, tgt, serializeForm);
+            return;
+        }
     case no_deserialize:
-        if (isDummySerializeDeserialize(expr))
-            doBuildDataset(ctx, expr->queryChild(0)->queryChild(0), tgt, format);
-        else
-            buildDeserializedDataset(ctx, expr->queryType(), expr->queryChild(0), tgt, expr->queryChild(2)->queryName());
-        return;
+        {
+            IHqlExpression * serialized = expr->queryChild(0);
+            _ATOM serializeForm = expr->queryChild(2)->queryName();
+            if (isDummySerializeDeserialize(expr))
+                doBuildDataset(ctx, serialized->queryChild(0), tgt, format);
+            else if (!typeRequiresDeserialization(expr->queryType(), serializeForm))
+                //Optimize creating a deserialized version of a dataset if the record is the same serialized and unserialized
+                buildDataset(ctx, serialized, tgt, FormatNatural);
+            else
+                buildDeserializedDataset(ctx, expr->queryType(), serialized, tgt, serializeForm);
+            return;
+        }
     case no_datasetfromrow:
         {
             IHqlExpression * row = expr->queryChild(0);