HPCC-9000 Variable length record display issues

Records with no maxlength but with records > 16k in size may display garbage
in ECLWatch.

This fix detects such cases and displays an error instead, and at the same
time raises the limit to 64k.

A full fix will require more work.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvdisksource.cpp

@@ -251,7 +251,11 @@ bool DirectDiskDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
     physical.readData(out, offset, returnedMeta->getMaxRecordSize());
     if (out.length() == 0)
         return false;
-    out.setLength(returnedMeta->getRecordSize(out.toByteArray()));
+    size32_t actualLength = returnedMeta->getRecordSize(out.toByteArray());
+    if (actualLength > readBlockSize)
+        throwError(FVERR_RowTooLarge);
+
+    out.setLength(actualLength);
     return true;
 }
 

common/fileview2/fverror.hpp

@@ -52,6 +52,7 @@
 #define FVERR_BadStringTermination              2026
 #define FVERR_CannotBrowseFile                  2027
 #define FVERR_PluginMismatch                    2028
+#define FVERR_RowTooLarge                       2029
 
 #define FVERR_CouldNotResolveX_Text             "Could not resolve file '%s' in DFS"
 #define FVERR_NoRecordDescription_Text          "DFS did not contain record description for '%s'"
@@ -81,5 +82,6 @@
 #define FVERR_UnrecognisedMappingFunctionXY_Text    "Unrecognised field mapping function %s.%s"
 #define FVERR_BadStringTermination_Text          "String not terminated correctly %.*s"
 #define FVERR_CannotBrowseFile_Text              "Cannot browse file '%s'"
+#define FVERR_RowTooLarge_Text                   "Row too large"
 
 #endif

common/fileview2/fvsource.cpp

@@ -24,6 +24,7 @@
 
 #include "fileview.hpp"
 #include "fvsource.ipp"
+#include "fverror.hpp"
 #include "hqlerror.hpp"
 #include "eclhelper.hpp"
 #include "hqlattr.hpp"
@@ -682,6 +683,8 @@ VariableRowBlock::VariableRowBlock(MemoryBuffer & _buffer, __int64 _start, __int
     {
         rowIndex.append(cur);
         cur += recordSize->getRecordSize(max-cur, buff + cur);
+        if (cur > max)
+            throwError(FVERR_RowTooLarge);
     }
     buffer.setLength(cur);
     rowIndex.append(cur);

common/fileview2/fvsource.ipp

@@ -25,7 +25,7 @@
 
 //Following constants configure different sizes etc.
 
-#define DISK_BLOCK_SIZE     8096            // Size of chunks read directly from file.
+#define DISK_BLOCK_SIZE     0x10000         // Size of chunks read directly from file.
 #define PAGED_WU_LIMIT      0x20000         // Page load work unit results >= this size.
 #define WU_BLOCK_SIZE       0x4000          // Size of chunks read from Work unit
 #define DISKREAD_PAGE_SIZE  200             // Number of rows to read in each chunk from file.