Merge pull request #10271 from richardkchapman/ecl-record-info

HPCC-17188 Store index payload information in the dali record

ecl/hql/hqlattr.cpp

@@ -1398,7 +1398,7 @@ IHqlExpression * HqlUnadornedNormalizer::createTransformed(IHqlExpression * expr
             ForEachChild(idx, expr)
             {
                 IHqlExpression * cur = expr->queryChild(idx);
-                if (cur->isAttribute())
+                if (cur->isAttribute() && (cur->queryName() != _payload_Atom))
                 {
                     IHqlExpression * mapped = transform(cur);
                     children.append(*mapped);

ecl/hql/hqlgram2.cpp

@@ -8291,6 +8291,9 @@ void HqlGram::expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHq
             break;
         case no_ifblock:
             lastFieldType = NULL;
+            // MORE - it might make sense to annotate the fields here (and below) - i.e.
+            // fields.append(*appendOwnedOperand(cur, createAttribute(_payload_Atom)));
+            // But that causes some changes to crcs, new errors about incompatible rows, etc
             fields.append(*LINK(cur));
             break;
         case no_field:
@@ -8312,6 +8315,9 @@ void HqlGram::expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHq
                 else
                 {
                     lastFieldType = cur->queryType();
+                    // MORE - it might make sense to annotate the fields here (and above) - i.e.
+                    // fields.append(*appendOwnedOperand(cur, createAttribute(_payload_Atom)));
+                    // But that causes some changes to crcs, new errors about incompatible rows, etc
                     fields.append(*LINK(cur));
                 }
                 break;

ecl/hql/hqlthql.cpp

@@ -1186,6 +1186,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             //First output the attributes...
             //MORE: Add attributes to the record definition
             bool first = true;
+            bool firstPayload = true;
             ForEachChild(idx, expr)
             {
                 IHqlExpression *child = queryChild(expr, idx);
@@ -1196,11 +1197,29 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                     if (fieldsInline)
                     {
                         if (!first)
-                            s.append(",");
+                        {
+                            if (firstPayload && child->hasAttribute(_payload_Atom))
+                            {
+                                s.append(" =>");
+                                firstPayload = false;
+                            }
+                            else
+                                s.append(",");
+                        }
                         s.append(" ");
                     }
                     else
+                    {
+                        if (!first)
+                        {
+                            if (firstPayload && child->hasAttribute(_payload_Atom))
+                            {
+                                s.remove(s.length()-2,2).append("\n").pad(indent).append("=>\n");
+                                firstPayload = false;
+                            }
+                        }
                         s.pad(indent);
+                    }
                     toECL(child, s, false, inType, idx+1);
                     if (!fieldsInline)
                         s.append(";\n");

ecl/hql/hqlutil.cpp

@@ -6390,6 +6390,29 @@ void TempTableTransformer::reportWarning(WarnErrorCategory category, IHqlExpress
     errorProcessor.reportWarning(category, code, errorMsg.str(), str(where->sourcePath), where->lineno, where->column, where->position);
 }
 
+IHqlExpression *notePayloadFields(IHqlExpression *record, unsigned payloadCount)
+{
+    HqlExprArray fields;
+    unwindChildren(fields, record);
+    unsigned idx = fields.length();
+    while (idx && payloadCount)
+    {
+        IHqlExpression * cur = &fields.item(--idx);
+        switch (cur->getOperator())
+        {
+        case no_record:
+            throwUnexpected();
+            break;
+        case no_field:
+        case no_ifblock:
+            fields.replace(*appendOwnedOperand(cur, createAttribute(_payload_Atom)), idx);
+            payloadCount--;
+            break;
+        }
+    }
+    return createRecord(fields);
+}
+
 IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record)
 {
     IHqlExpression * payload = record->queryAttribute(_payload_Atom);

ecl/hql/hqlutil.hpp

@@ -521,6 +521,7 @@ extern HQL_API IHqlExpression * extractCppBodyAttrs(unsigned len, const char * v
 extern HQL_API unsigned cleanupEmbeddedCpp(unsigned len, char * buffer);
 extern HQL_API bool isNullList(IHqlExpression * expr);
 
+extern HQL_API IHqlExpression *notePayloadFields(IHqlExpression *record, unsigned payloadCount);
 extern HQL_API IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record);
 extern HQL_API IHqlExpression *getDictionarySearchRecord(IHqlExpression *record);
 extern HQL_API IHqlExpression * createSelectMapRow(IErrorReceiver & errorProcessor, ECLlocation & location, IHqlExpression * dict, IHqlExpression *values);

ecl/hqlcpp/hqlcpp.ipp

@@ -1713,7 +1713,7 @@ public:
     void buildHTTPtoXml(BuildCtx & ctx);
     void buildSOAPtoXml(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * transform, IHqlExpression * selSeq);
 
-    void buildRecordEcl(BuildCtx & subctx, IHqlExpression * dataset, const char * methodName, bool removeXpath);
+    void buildRecordEcl(BuildCtx & subctx, IHqlExpression * dataset, const char * methodName);
     void doTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self);
     void doUpdateTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self, BoundRow * previous, bool alwaysNextRow);
     void doInlineTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * targetRow);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9920,15 +9920,11 @@ void HqlCppTranslator::buildClusterHelper(BuildCtx & ctx, IHqlExpression * expr)
 }
 
 
-void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * dataset, const char * methodName, bool removeXpath)
+void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * record, const char * methodName)
 {
     StringBuffer eclFuncName;
     StringBuffer s;
 
-    //Ensure the ECL for the record reflects its serialized form, not the internal form
-    OwnedHqlExpr record = getSerializedForm(dataset->queryRecord(), diskAtom);
-    if (removeXpath)
-        record.setown(removeAttributeFromFields(record, xpathAtom));
     appendUniqueId(eclFuncName.append("ecl"), getConsistentUID(record));
 
     BuildCtx declarectx(*code, declareAtom);
@@ -10320,6 +10316,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
     buildUpdateHelper(instance->createctx, *instance, dataset, updateAttr);
     buildClusterHelper(instance->classctx, expr);
 
+    LinkedHqlExpr serializedRecord = record;
+
     // virtual unsigned getKeyedSize()
     HqlExprArray fields;
     unwindChildren(fields, record);
@@ -10327,12 +10325,16 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
     fields.popn(numPayloadFields(expr));
     OwnedHqlExpr keyedRecord = createRecord(fields); // must be fixed length => no maxlength
     if (expr->hasAttribute(_payload_Atom))
+    {
         instance->classctx.addQuoted(s.clear().append("virtual unsigned getKeyedSize() { return ").append(getFixedRecordSize(keyedRecord)).append("; }"));
+        serializedRecord.setown(notePayloadFields(serializedRecord, numPayloadFields(expr)));
+    }
     else
         instance->classctx.addQuoted(s.clear().append("virtual unsigned getKeyedSize() { return (unsigned) -1; }"));
 
     //virtual const char * queryRecordECL() = 0;
-    buildRecordEcl(instance->createctx, dataset, "queryRecordECL", false);
+    serializedRecord.setown(getSerializedForm(serializedRecord, diskAtom));
+    buildRecordEcl(instance->createctx, serializedRecord, "queryRecordECL");
 
     doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
     HqlExprArray xmlnsAttrs;
@@ -10636,7 +10638,11 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
                 doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
 
             //virtual const char * queryRecordECL() = 0;
-            buildRecordEcl(instance->createctx, dataset, "queryRecordECL", expr->hasAttribute(noXpathAtom));
+            //Ensure the ECL for the record reflects its serialized form, not the internal form
+            OwnedHqlExpr record = getSerializedForm(dataset->queryRecord(), diskAtom);
+            if (expr->hasAttribute(noXpathAtom))
+                record.setown(removeAttributeFromFields(record, xpathAtom));
+            buildRecordEcl(instance->createctx, record, "queryRecordECL");
 
             buildExpiryHelper(instance->createctx, expireAttr);
             buildUpdateHelper(instance->createctx, *instance, dataset, updateAttr);