Merge pull request #12963 from ghalliday/issue21820

HPCC-21820 Add options to generate record formats to the workunit

Reviewed-By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/wuattr.cpp

@@ -147,6 +147,8 @@ const static WuAttrInfo attrInfo[] = {
     ATTRX(State, SMeasureText),
     ATTRX(Cluster, SMeasureText),
     ATTRX(CriticalSection, SMeasureText),
+    CHILD(DiskFormat, SMeasureText, "diskFormat"),
+    CHILD(RecordFormat, SMeasureText, "recordFormat"),
     { WaMax, SMeasureNone, nullptr, nullptr, nullptr, nullptr, nullptr, WaNone, WaNone }
 };
 

common/workunit/wuattr.hpp

@@ -132,6 +132,8 @@ enum WuAttr : unsigned
     WaState,
     WaCluster,
     WaCriticalSection,
+    WaDiskFormat,
+    WaRecordFormat,
     WaMax
 };
 inline WuAttr & operator++(WuAttr & x) { assert(x<WaMax); x = (WuAttr)(x+1); return x; }

ecl/hqlcpp/hqlcpp.cpp

@@ -1856,6 +1856,8 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.checkDuplicateMinActivities, "checkDuplicateMinActivities", 100),
         DebugOption(options.diskReadsAreSimple, "diskReadsAreSimple", false), // Not yet enabled - needs filters to default to generating keyed info first
         DebugOption(options.genericDiskReads, "genericDiskReads", false),
+        DebugOption(options.generateActivityFormats, "generateActivityFormats", false),
+        DebugOption(options.generateDiskFormats, "generateDiskFormats", false),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -825,6 +825,8 @@ struct HqlCppOptions
     bool                newIndexReadMapping;
     bool                diskReadsAreSimple;
     bool                genericDiskReads;
+    bool                generateActivityFormats;
+    bool                generateDiskFormats;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -1763,6 +1765,7 @@ public:
     void doInlineTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * targetRow);
     void doUserTransform(BuildCtx & ctx, IHqlExpression * transform, BoundRow * self);
     IHqlExpression * createOrderFromSortList(const DatasetReference & dataset, IHqlExpression * sortList, IHqlExpression * leftSelect, IHqlExpression * rightSelect);
+    void addFormatAttribute(ActivityInstance & instance, WuAttr attr, IHqlExpression * record);
 
     void buildSkewThresholdMembers(BuildCtx & ctx, IHqlExpression * expr);
     void doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, const HqlExprArray & left, const HqlExprArray & right);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -2197,7 +2197,7 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
                 addAttribute(WaMetaGroupSortOrder, getExprECL(groupSortOrder, s.clear(), true).str());
         }
 
-        if (options.noteRecordSizeInGraph)
+        if (options.noteRecordSizeInGraph || options.generateActivityFormats)
         {
             LinkedHqlExpr record = activityExpr->queryRecord();
             if (!record && (getNumChildTables(activityExpr) == 1))
@@ -2207,9 +2207,15 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
                 //In Thor the serialized record is the interesting value, so include that in the graph
                 if (translator.targetThor())
                     record.setown(getSerializedForm(record, diskAtom));
-                StringBuffer temp;
-                getRecordSizeText(temp, record);
-                addAttribute(WaRecordSize, temp.str());
+
+                if (options.noteRecordSizeInGraph)
+                {
+                    StringBuffer temp;
+                    getRecordSizeText(temp, record);
+                    addAttribute(WaRecordSize, temp.str());
+                }
+                if (options.generateActivityFormats)
+                    translator.addFormatAttribute(*this, WaRecordFormat, record);
             }
         }
 
@@ -10226,6 +10232,14 @@ void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * record
 }
 
 
+void HqlCppTranslator::addFormatAttribute(ActivityInstance & instance, WuAttr attr, IHqlExpression * record)
+{
+    //MORE: Better would be to add a section to the workunit and only generate the record formats once, and reference them from here
+    StringBuffer recordEcl;
+    getRecordECL(record, recordEcl);
+    instance.addAttribute(attr, recordEcl.str());
+}
+
 void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name, bool removeFilepos, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta)
 {
     IHqlExpression * payload = expr ? expr->queryAttribute(_payload_Atom) : NULL;

ecl/hqlcpp/hqlsource.cpp

@@ -1186,6 +1186,9 @@ void SourceBuilder::buildReadMembers(IHqlExpression * expr)
     //---- virtual bool transformMayFilter() { return <bool>; } ----
     if (transformCanFilter)
         translator.doBuildBoolFunction(instance->classctx, "transformMayFilter", true);
+
+    if (translator.queryOptions().generateDiskFormats)
+        translator.addFormatAttribute(*instance, WaDiskFormat, tableExpr->queryRecord());
 }
 
 void SourceBuilder::buildLimits(BuildCtx & classctx, IHqlExpression * expr, unique_id_t id)