Merge pull request #10671 from richardkchapman/generate-projected-diskmeta

HPCC-18780 Generate projected as well as expected disk meta functions

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/thorhelper/thorfile.cpp

@@ -53,6 +53,10 @@ public:
     {
         return (IOutputMetaData *)recordSize;
     }
+    virtual IOutputMetaData * queryProjectedDiskRecordSize()
+    {
+        return (IOutputMetaData *)recordSize;
+    }
     virtual unsigned getFormatCrc()
     {
         return 0;

ecl/hqlcpp/hqlckey.cpp

@@ -1260,9 +1260,12 @@ void KeyedJoinInfo::splitFilter(IHqlExpression * filter, SharedHqlExpr & keyTarg
 void HqlCppTranslator::buildKeyedJoinExtra(ActivityInstance & instance, IHqlExpression * expr, KeyedJoinInfo * info)
 {
     //virtual IOutputMetaData * queryDiskRecordSize() = 0;
+    //virtual IOutputMetaData * queryProjectedDiskRecordSize() = 0;
     if (info->isFullJoin())
+    {
         buildMetaMember(instance.classctx, info->queryRawRhs(), false, "queryDiskRecordSize");
-
+        buildMetaMember(instance.classctx, info->queryRawRhs(), false, "queryProjectedDiskRecordSize");
+    }
     //virtual unsigned __int64 extractPosition(const void * _right) = 0;  // Gets file position value from rhs row
     if (info->isFullJoin())
     {
@@ -1328,6 +1331,7 @@ void HqlCppTranslator::buildKeyJoinIndexReadHelper(ActivityInstance & instance,
     bool hasFilePosition = getBoolAttribute(indexExpr, filepositionAtom, true);
     serializedRecord.setown(createMetadataIndexRecord(serializedRecord, hasFilePosition));
     buildMetaMember(instance.classctx, serializedRecord, false, "queryIndexRecordSize");
+    buildMetaMember(instance.classctx, serializedRecord, false, "queryProjectedIndexRecordSize");
 
     //virtual void createSegmentMonitors(IIndexReadContext *ctx, const void *lhs) = 0;
     info->buildMonitors(instance.startctx);

ecl/hqlcpp/hqlsource.cpp

@@ -2023,6 +2023,7 @@ ABoundActivity * SourceBuilder::buildActivity(BuildCtx & ctx, IHqlExpression * e
                 bool hasFilePosition = getBoolAttribute(indexExpr, filepositionAtom, true);
                 serializedRecord.setown(createMetadataIndexRecord(serializedRecord, hasFilePosition));
                 translator.buildMetaMember(instance->classctx, serializedRecord, false, "queryDiskRecordSize");
+                translator.buildMetaMember(instance->classctx, serializedRecord, false, "queryProjectedDiskRecordSize");
                 break;
             }
             default:
@@ -2030,10 +2031,14 @@ ABoundActivity * SourceBuilder::buildActivity(BuildCtx & ctx, IHqlExpression * e
                 {
                     OwnedHqlExpr diskTable = createDataset(no_anon, LINK(physicalRecord));
                     translator.buildMetaMember(instance->classctx, diskTable, false, "queryDiskRecordSize");
+                    if (activityKind != TAKpiperead)
+                        translator.buildMetaMember(instance->classctx, diskTable, false, "queryProjectedDiskRecordSize");
                 }
                 else
                 {
                     translator.buildMetaMember(instance->classctx, tableExpr, isGrouped(tableExpr), "queryDiskRecordSize");
+                    if (activityKind != TAKpiperead)
+                        translator.buildMetaMember(instance->classctx, tableExpr, isGrouped(tableExpr), "queryProjectedDiskRecordSize");
                 }
             }
         }
@@ -6973,6 +6978,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityXmlRead(BuildCtx & ctx, IHqlEx
     doBuildVarStringFunction(instance->classctx, "getXmlIteratorPath", queryRealChild(mode, 0));
 
     buildMetaMember(instance->classctx, tableExpr, false, "queryDiskRecordSize");  // A lie, but I don't care....
+    buildMetaMember(instance->classctx, tableExpr, false, "queryProjectedDiskRecordSize");  // A lie, but I don't care....
 
     //virtual unsigned getFlags() = 0;
     StringBuffer flags;

rtl/eclrtl/eclhelper_base.cpp

@@ -329,6 +329,7 @@ ICompare * CThorSubSortArg::queryCompareSerializedRow() { return NULL; }
 bool CThorKeyedJoinArg::diskAccessRequired() { return false; }
 const char * CThorKeyedJoinArg::getFileName() { return NULL; }
 IOutputMetaData * CThorKeyedJoinArg::queryDiskRecordSize() { return NULL; }
+IOutputMetaData * CThorKeyedJoinArg::queryProjectedDiskRecordSize() { return NULL; }
 unsigned __int64 CThorKeyedJoinArg::extractPosition(const void * _right) { return 0; }
 
 bool CThorKeyedJoinArg::leftCanMatch(const void * inputRow) { return true; }

rtl/eclrtl/eclhelper_dyn.cpp

@@ -232,6 +232,10 @@ public:
     {
         return in;
     }
+    virtual IOutputMetaData * queryProjectedDiskRecordSize() override final
+    {
+        return in;
+    }
     virtual unsigned getFormatCrc() override
     {
         return 0;  // engines should treat 0 as 'ignore'
@@ -297,6 +301,10 @@ public:
     {
         return in;
     }
+    virtual IOutputMetaData * queryProjectedDiskRecordSize() override final
+    {
+        return in;
+    }
     virtual unsigned getFormatCrc() override
     {
         return 0;  // engines should treat 0 as 'ignore'

rtl/include/eclhelper.hpp

@@ -1680,7 +1680,8 @@ struct IHThorFetchContext
 {
     virtual unsigned __int64 extractPosition(const void * _right) = 0;  // Gets file position value from rhs row
     virtual const char * getFileName() = 0;                 // Returns filename of raw file fpos'es refer into
-    virtual IOutputMetaData * queryDiskRecordSize() = 0;        // Returns record size of raw file fpos'es refer into
+    virtual IOutputMetaData * queryDiskRecordSize() = 0;            // Expected layout
+    virtual IOutputMetaData * queryProjectedDiskRecordSize() = 0;   // Projected layout
     virtual unsigned getFetchFlags() { return 0; }              
     virtual unsigned getDiskFormatCrc() { return 0; }
     virtual void getFileEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }
@@ -1695,7 +1696,8 @@ struct IHThorKeyedJoinBaseArg : public IHThorArg
 
     // Inside the indexRead remote activity:
     virtual const char * getIndexFileName() = 0;
-    virtual IOutputMetaData * queryIndexRecordSize() = 0;
+    virtual IOutputMetaData * queryIndexRecordSize() = 0;            // Expected layout
+    virtual IOutputMetaData * queryProjectedIndexRecordSize() = 0;   // Projected layout
     virtual void createSegmentMonitors(IIndexReadContext *ctx, const void *lhs) = 0;
     virtual bool indexReadMatch(const void * indexRow, const void * inputRow, unsigned __int64 keyedFpos, IBlobProvider * blobs) = 0;
     virtual unsigned getJoinLimit() = 0;                                        // if a key joins more than this limit no records are output (0 = no limit)
@@ -2277,7 +2279,8 @@ struct IHThorCompoundBaseArg : public IHThorArg
 struct IHThorIndexReadBaseArg : extends IHThorCompoundBaseArg
 {
     virtual const char * getFileName() = 0;
-    virtual IOutputMetaData * queryDiskRecordSize() = 0;                // size of records on disk may differ if records are transformed on read
+    virtual IOutputMetaData * queryDiskRecordSize() = 0;            // Expected layout
+    virtual IOutputMetaData * queryProjectedDiskRecordSize() = 0;   // Projected layout
     virtual unsigned getFlags() = 0;
     virtual unsigned getFormatCrc() = 0;
     virtual void setCallback(IThorIndexCallback * callback) = 0;
@@ -2290,7 +2293,8 @@ struct IHThorIndexReadBaseArg : extends IHThorCompoundBaseArg
 struct IHThorDiskReadBaseArg : extends IHThorCompoundBaseArg
 {
     virtual const char * getFileName() = 0;
-    virtual IOutputMetaData * queryDiskRecordSize() = 0;                // size of records on disk may differ if records are transformed on read
+    virtual IOutputMetaData * queryDiskRecordSize() = 0;            // Expected layout
+    virtual IOutputMetaData * queryProjectedDiskRecordSize() = 0;   // Projected layout
     virtual unsigned getFlags() = 0;
     virtual unsigned getFormatCrc() = 0;
     virtual void getEncryptKey(size32_t & keyLen, void * & key) { keyLen = 0; key = 0; }

rtl/include/eclhelper_base.hpp

@@ -441,6 +441,7 @@ class ECLRTL_API CThorKeyedJoinArg : public CThorArgOf<IHThorKeyedJoinArg>
     virtual bool diskAccessRequired() override;
     virtual const char * getFileName() override;
     virtual IOutputMetaData * queryDiskRecordSize() override;
+    virtual IOutputMetaData * queryProjectedDiskRecordSize() override;
     virtual unsigned __int64 extractPosition(const void * _right) override;
     
     // For the data going to the indexRead remote activity:
@@ -1228,7 +1229,8 @@ public:
 
     virtual unsigned getFlags()                             { return TDXtemporary|TDXcompress; }
     virtual unsigned getFormatCrc()                         { rtlFailUnexpected(); return 0; }
-    virtual IOutputMetaData * queryDiskRecordSize()             { return meta; }
+    virtual IOutputMetaData * queryDiskRecordSize()         { return meta; }
+    virtual IOutputMetaData * queryProjectedDiskRecordSize() { return meta; }
     virtual IOutputMetaData * queryOutputMeta()             { return meta; }
     virtual const char * getFileName()                      { return filename; }
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _left) { rtlFailUnexpected(); return 0; }

thorlcr/activities/csvread/thcsvrslave.cpp

@@ -299,7 +299,7 @@ class CCsvReadSlaveActivity : public CDiskReadSlaveActivityBase
         }
     }
 public:
-    CCsvReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container)
+    CCsvReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container, nullptr)
     {
         helper = static_cast <IHThorCsvReadArg *> (queryHelper());
         stopAfter = (rowcount_t)helper->getChooseNLimit();

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -74,10 +74,8 @@ protected:
 
 public:
     CDiskReadSlaveActivityRecord(CGraphElementBase *_container, IHThorArg *_helper=NULL) 
-        : CDiskReadSlaveActivityBase(_container)
+        : CDiskReadSlaveActivityBase(_container, _helper)
     {
-        if (_helper)
-            baseHelper.set(_helper);
         helper = (IHThorDiskReadArg *)queryHelper();
         IOutputMetaData *diskRowMeta = queryDiskRowInterfaces()->queryRowMetaData()->querySerializedDiskMeta();
         isFixedDiskWidth = diskRowMeta->isFixedSize();

thorlcr/activities/thdiskbaseslave.cpp

@@ -202,8 +202,10 @@ const char * CDiskPartHandlerBase::queryLogicalFilename(const void * row)
 
 //////////////////////////////////////////////
 
-CDiskReadSlaveActivityBase::CDiskReadSlaveActivityBase(CGraphElementBase *_container) : CSlaveActivity(_container)
+CDiskReadSlaveActivityBase::CDiskReadSlaveActivityBase(CGraphElementBase *_container, IHThorArg *_helper) : CSlaveActivity(_container)
 {
+    if (_helper)
+        baseHelper.set(_helper);
     helper = (IHThorDiskReadBaseArg *)queryHelper();
     reInit = 0 != (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename));
     crcCheckCompressed = getOptBool(THOROPT_READCOMPRESSED_CRC, false);

thorlcr/activities/thdiskbaseslave.ipp

@@ -91,7 +91,7 @@ protected:
     rowcount_t diskProgress = 0;
 
 public:
-    CDiskReadSlaveActivityBase(CGraphElementBase *_container);
+    CDiskReadSlaveActivityBase(CGraphElementBase *_container, IHThorArg *_helper);
     const char *queryLogicalFilename(unsigned index);
     IThorRowInterfaces * queryDiskRowInterfaces();
     virtual void start() override;

thorlcr/activities/xmlread/thxmlreadslave.cpp

@@ -204,7 +204,7 @@ class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase
         }
     };
 public:
-    CXmlReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container)
+    CXmlReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container, nullptr)
     {
         out = NULL;
         helper = (IHThorXmlReadArg *)queryHelper();