Merge branch 'candidate-7.0.x' into candidate-7.2.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/SprayVariable.xml

@@ -13,34 +13,34 @@
     </indexterm><indexterm>
       <primary>SprayDelimited</primary>
     </indexterm>(</emphasis> <emphasis> sourceIP </emphasis> <emphasis
-  role="bold">, </emphasis> <emphasis>sourcepath , </emphasis> <emphasis
-  role="bold">[</emphasis> <emphasis> maxrecordsize </emphasis> <emphasis
+  role="bold">, </emphasis> <emphasis>sourcePath , </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis> sourceMaxRecordSize </emphasis>
+  <emphasis role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis
+  role="bold"> [</emphasis> <emphasis> sourceCsvSeparate </emphasis> <emphasis
+  role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis> sourceCsvTerminate </emphasis> <emphasis
+  role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis> sourceCsvQuote </emphasis> <emphasis
   role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis role="bold">
-  [</emphasis> <emphasis> srcCSVseparate </emphasis> <emphasis role="bold">]
-  </emphasis> <emphasis>, </emphasis> <emphasis role="bold">[</emphasis>
-  <emphasis> srcCSVterminate </emphasis> <emphasis role="bold">] </emphasis>
-  <emphasis>, </emphasis> <emphasis role="bold">[</emphasis> <emphasis>
-  srcCSVquote </emphasis> <emphasis role="bold">] </emphasis> <emphasis>,
-  </emphasis> <emphasis role="bold"> </emphasis> <emphasis>destinationgroup,
-  destinationlogicalname </emphasis>, <emphasis role="bold">[</emphasis>
-  <emphasis>timeout</emphasis> <emphasis role="bold">]</emphasis> ,<emphasis
-  role="bold"> [</emphasis> <emphasis>espserverIPport</emphasis> <emphasis
-  role="bold">]</emphasis> ,<emphasis> </emphasis> <emphasis
-  role="bold">[</emphasis> <emphasis>maxConnections</emphasis> <emphasis
+  </emphasis> <emphasis>destinationGroup, destinationLogicalName </emphasis>,
+  <emphasis role="bold">[</emphasis> <emphasis>timeout</emphasis> <emphasis
   role="bold">]</emphasis> ,<emphasis role="bold"> [</emphasis>
-  <emphasis>allowoverwrite</emphasis> <emphasis role="bold">] , [</emphasis>
-  <emphasis>replicate</emphasis> <emphasis role="bold">] , [</emphasis>
-  <emphasis> compress </emphasis><emphasis role="bold">]</emphasis> ,
-  <emphasis role="bold"> [</emphasis> <emphasis> sourceCsvEscape </emphasis>
-  <emphasis role="bold">]</emphasis>, <emphasis role="bold"> [</emphasis>
-  <emphasis> failIfNoSourceFile </emphasis> <emphasis
+  <emphasis>espServerIpPort</emphasis> <emphasis role="bold">]</emphasis>
+  ,<emphasis> </emphasis> <emphasis role="bold">[</emphasis>
+  <emphasis>maxConnections</emphasis> <emphasis role="bold">]</emphasis>
+  ,<emphasis role="bold"> [</emphasis> <emphasis>allowOverwrite</emphasis>
+  <emphasis role="bold">] , [</emphasis> <emphasis>replicate</emphasis>
+  <emphasis role="bold">] , [</emphasis> <emphasis>compress
+  </emphasis><emphasis role="bold">]</emphasis> , <emphasis role="bold">
+  [</emphasis> <emphasis> sourceCsvEscape </emphasis> <emphasis
   role="bold">]</emphasis>, <emphasis role="bold"> [</emphasis> <emphasis>
-  recordStructurePresent </emphasis> <emphasis role="bold">]</emphasis>,
-  <emphasis role="bold"> [</emphasis> <emphasis> quotedTerminator </emphasis>
+  failIfNoSourceFile </emphasis> <emphasis role="bold">]</emphasis>, <emphasis
+  role="bold"> [</emphasis> <emphasis> recordStructurePresent </emphasis>
   <emphasis role="bold">]</emphasis>, <emphasis role="bold"> [</emphasis>
-  <emphasis> encoding </emphasis> <emphasis role="bold">]</emphasis><emphasis
-  role="bold">[</emphasis> <emphasis>expireDays</emphasis> <emphasis
-  role="bold">])</emphasis></para>
+  <emphasis> quotedTerminator </emphasis> <emphasis role="bold">]</emphasis>,
+  <emphasis role="bold"> [</emphasis> <emphasis> encoding </emphasis>
+  <emphasis role="bold">]</emphasis><emphasis role="bold">[</emphasis>
+  <emphasis>expireDays</emphasis> <emphasis role="bold">])</emphasis></para>
 
   <para><emphasis>dfuwuid</emphasis> <emphasis role="bold"> :=
   STD.File.fSprayDelimited<indexterm>
@@ -52,23 +52,23 @@
     </indexterm><indexterm>
       <primary>fSprayDelimited</primary>
     </indexterm>(</emphasis> <emphasis> sourceIP </emphasis> <emphasis
-  role="bold">, </emphasis> <emphasis>sourcepath , </emphasis> <emphasis
-  role="bold"> [</emphasis> <emphasis> maxrecordsize </emphasis> <emphasis
-  role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis
-  role="bold">[</emphasis> <emphasis> srcCSVseparate </emphasis> <emphasis
+  role="bold">, </emphasis> <emphasis>sourcePath , </emphasis> <emphasis
+  role="bold"> [</emphasis> <emphasis> sourceMaxRecordSize </emphasis>
+  <emphasis role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis> sourceCsvSeparate </emphasis> <emphasis
   role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis
-  role="bold">[</emphasis> <emphasis> srcCSVterminate </emphasis> <emphasis
+  role="bold">[</emphasis> <emphasis> sourceCsvTerminate </emphasis> <emphasis
   role="bold">] </emphasis> <emphasis>, </emphasis> <emphasis role="bold">
-  [</emphasis> <emphasis> srcCSVquote </emphasis> <emphasis role="bold">]
-  </emphasis> <emphasis>, destinationgroup, destinationlogicalname </emphasis>
+  [</emphasis> <emphasis> sourceCsvQuote </emphasis> <emphasis role="bold">]
+  </emphasis> <emphasis>, destinationGroup, destinationLogicalName </emphasis>
   <emphasis role="bold">, [</emphasis> <emphasis>timeout</emphasis> <emphasis
-  role="bold">] , [</emphasis> <emphasis>espserverIPport</emphasis> <emphasis
+  role="bold">] , [</emphasis> <emphasis>espServerIpPort</emphasis> <emphasis
   role="bold">]</emphasis> ,<emphasis> </emphasis> <emphasis
   role="bold">[</emphasis> <emphasis>maxConnections</emphasis> <emphasis
   role="bold">]</emphasis> ,<emphasis role="bold"> [</emphasis>
-  <emphasis>allowoverwrite</emphasis> <emphasis role="bold">] , [</emphasis>
+  <emphasis>allowOverwrite</emphasis> <emphasis role="bold">] , [</emphasis>
   <emphasis>replicate</emphasis> <emphasis role="bold">] , [</emphasis>
-  <emphasis> compress </emphasis> <emphasis role="bold">]</emphasis> ,
+  <emphasis>compress </emphasis> <emphasis role="bold">]</emphasis> ,
   <emphasis role="bold"> [</emphasis> <emphasis> sourceCsvEscape </emphasis>
   <emphasis role="bold">]</emphasis>, <emphasis role="bold"> [</emphasis>
   <emphasis> failIfNoSourceFile </emphasis> <emphasis
@@ -82,7 +82,7 @@
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">
-      <colspec colwidth="80.50pt" />
+      <colspec colwidth="90.50pt" />
 
       <colspec />
 
@@ -95,49 +95,49 @@
         </row>
 
         <row>
-          <entry><emphasis>sourcepath</emphasis></entry>
+          <entry><emphasis>sourcePath</emphasis></entry>
 
           <entry>A null-terminated string containing the path and name of the
           file.</entry>
         </row>
 
         <row>
-          <entry><emphasis>maxrecordsize</emphasis></entry>
+          <entry><emphasis>sourceMaxRecordSize</emphasis></entry>
 
           <entry>Optional. An integer containing the maximum size of the
           records in the file. If omitted, the default is 4096.</entry>
         </row>
 
         <row>
-          <entry><emphasis>srcCSVseparate</emphasis></entry>
+          <entry><emphasis>sourceCsvSeparate</emphasis></entry>
 
           <entry>Optional. A null-terminated string containing the CSV field
           separator. If omitted, the default is '\\,'</entry>
         </row>
 
         <row>
-          <entry><emphasis>srcCSVterminate</emphasis></entry>
+          <entry><emphasis>sourceCSVterminate</emphasis></entry>
 
           <entry>Optional. A null-terminated string containing the CSV record
           separator. If omitted, the default is '\\n,\\r\\n'</entry>
         </row>
 
         <row>
-          <entry><emphasis>srcCSVquote</emphasis></entry>
+          <entry><emphasis>sourceCSVquote</emphasis></entry>
 
           <entry>Optional. A null-terminated string containing the CSV quoted
           field delimiter. If omitted, the default is '\"'</entry>
         </row>
 
         <row>
-          <entry><emphasis>destinationgroup</emphasis></entry>
+          <entry><emphasis>destinationGroup</emphasis></entry>
 
           <entry>A null-terminated string containing the name of the specific
           supercomputer within the target cluster.</entry>
         </row>
 
         <row>
-          <entry><emphasis>destinationlogicalname</emphasis></entry>
+          <entry><emphasis>destinationLogicalName</emphasis></entry>
 
           <entry>A null-terminated string containing the logical name of the
           file.</entry>
@@ -153,7 +153,7 @@
         </row>
 
         <row>
-          <entry><emphasis>espserverIPport</emphasis></entry>
+          <entry><emphasis>espServerIpPort</emphasis></entry>
 
           <entry>Optional. A null-terminated string containing the protocol,
           IP, port, and directory, or the DNS equivalent, of the ESP server
@@ -170,7 +170,7 @@
         </row>
 
         <row>
-          <entry><emphasis>allowoverwrite</emphasis></entry>
+          <entry><emphasis>allowOverwrite</emphasis></entry>
 
           <entry>Optional. A boolean TRUE or FALSE flag indicating whether to
           allow the new file to overwrite an existing file of the same name.

ecl/hthor/hthor.cpp

@@ -8294,13 +8294,14 @@ bool CHThorDiskReadBaseActivity::openNext()
             else
                 actualFilter.appendFilters(fieldFilters);
 
-            bool canSerializeTypeInfo = actualDiskMeta->queryTypeInfo()->canSerialize() && projectedDiskMeta->queryTypeInfo()->canSerialize();
+            bool tryRemoteStream = actualDiskMeta->queryTypeInfo()->canInterpret() && actualDiskMeta->queryTypeInfo()->canSerialize() &&
+                                   projectedDiskMeta->queryTypeInfo()->canInterpret() && projectedDiskMeta->queryTypeInfo()->canSerialize();
 
             /* If part can potentially be remotely streamed, 1st check if any part is local,
              * then try to remote stream, and otherwise failover to legacy remote access
              */
             unsigned localCopy = NotFound;
-            if (canSerializeTypeInfo && (rt_binary == readType))
+            if (tryRemoteStream && (rt_binary == readType))
             {
                 std::vector<unsigned> remoteCandidates;
                 // scan for local part 1st

ecllibrary/std/File.ecl

@@ -460,6 +460,9 @@ EXPORT SprayVariable(varstring sourceIP, varstring sourcePath, integer4 sourceMa
  * @param recordStructurePresent If TRUE derives the record structure from the header of the file.
  * @param quotedTerminator Can the terminator character be included in a quoted field.  Defaults to TRUE.
  *                      If FALSE it allows quicker partitioning of the file (avoiding a complete file scan).
+ * @param encoding      A null-terminated string containing the encoding. 
+ *                      Can be set to one of the following: 
+ *                      ascii, utf8, utf8n, utf16, utf16le, utf16be, utf32, utf32le,utf32be. If omitted, the default is ascii.
  * @param expireDays    Number of days to auto-remove file. Default is -1, not expire.
  * @return              The DFU workunit id for the job.
  */

rtl/include/eclhelper.hpp

@@ -427,6 +427,7 @@ struct RtlTypeInfo : public RtlITypeInfo
     inline bool isSigned() const { return (fieldType & RFTMunsigned) == 0; }
     inline bool isUnsigned() const { return (fieldType & RFTMunsigned) != 0; }
     inline bool isBlob() const { return getType() == type_blob; }
+    inline bool canInterpret() const { return (fieldType & RFTMcannotinterpret) == 0; }
     inline unsigned getDecimalDigits() const { return (length & 0xffff); }
     inline unsigned getDecimalPrecision() const { return (length >> 16); }
     inline unsigned getBitfieldIntSize() const { return (length & 0xff); }

system/jlib/jfile.cpp

@@ -6372,7 +6372,8 @@ public:
         }
 #elif defined (__linux__)
         ptr = (byte *) mmap(NULL, mapsz, writeaccess?(PROT_READ|PROT_WRITE):PROT_READ, MAP_SHARED|MAP_NORESERVE, hfile, realofs);
-            // error checking TBD
+        if (ptr == MAP_FAILED)
+            throw makeOsException(errno, "CMemoryMappedFile::reinit");
 #else
         UNIMPLEMENTED;
 #endif

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -267,12 +267,13 @@ void CDiskRecordPartHandler::open()
     IOutputMetaData *expectedFormat = activity.helper->queryDiskRecordSize();
     Owned<ITranslator> translator = activity.getTranslators(*partDesc);
     IOutputMetaData *actualFormat = translator ? &translator->queryActualFormat() : expectedFormat;
-    bool canSerializeTypeInfo = actualFormat->queryTypeInfo()->canSerialize() && projectedFormat->queryTypeInfo()->canSerialize();
+    bool tryRemoteStream = actualFormat->queryTypeInfo()->canInterpret() && actualFormat->queryTypeInfo()->canSerialize() &&
+                           projectedFormat->queryTypeInfo()->canInterpret() && projectedFormat->queryTypeInfo()->canSerialize();
 
     /* If part can potentially be remotely streamed, 1st check if any part is local,
      * then try to remote stream, and otherwise failover to legacy remote access
      */
-    if (canSerializeTypeInfo)
+    if (tryRemoteStream)
     {
         std::vector<unsigned> remoteCandidates;
         // scan for non remote candidate part 1st

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -158,7 +158,8 @@ public:
 
                 Owned<ITranslator> translator = getTranslators(part);
                 IOutputMetaData *actualFormat = translator ? &translator->queryActualFormat() : expectedFormat;
-                bool canSerializeTypeInfo = actualFormat->queryTypeInfo()->canSerialize() && projectedFormat->queryTypeInfo()->canSerialize();
+                bool tryRemoteStream = actualFormat->queryTypeInfo()->canInterpret() && actualFormat->queryTypeInfo()->canSerialize() &&
+                                       projectedFormat->queryTypeInfo()->canInterpret() && projectedFormat->queryTypeInfo()->canSerialize();
                 bool usesBlobs = 0 != (helper->getFlags() & TIRusesblob);
 
                 unsigned crc=0;
@@ -167,7 +168,7 @@ public:
                 /* If part can potentially be remotely streamed, 1st check if any part is local,
                  * then try to remote stream, and otherwise failover to legacy remote access
                  */
-                if (canSerializeTypeInfo && !usesBlobs && !localMerge)
+                if (tryRemoteStream && !usesBlobs && !localMerge)
                 {
                     std::vector<unsigned> remoteCandidates;
                     for (unsigned copy=0; copy<part.numCopies(); copy++)

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -1046,7 +1046,9 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
                 Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode);
                 if (translator)
                 {
-                    if (!publishedFormat->queryTypeInfo()->canSerialize() || !projectedFormat->queryTypeInfo()->canSerialize())
+                    bool canRemoteTranslate = publishedFormat->queryTypeInfo()->canInterpret() && publishedFormat->queryTypeInfo()->canSerialize() &&
+                                              projectedFormat->queryTypeInfo()->canInterpret() && projectedFormat->queryTypeInfo()->canSerialize();
+                    if (!canRemoteTranslate)
                         throw MakeActivityException(&activity, 0, "CKeyLookupRemoteHandler - translation required, but formats unserializable");
                     msg.append(static_cast<std::underlying_type<RecordTranslationMode>::type>(translationMode));
                     msg.append(publishedFormatCrc);
@@ -1341,7 +1343,9 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
                 Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryDiskRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode);
                 if (translator)
                 {
-                    if (!publishedFormat->queryTypeInfo()->canSerialize() || !projectedFormat->queryTypeInfo()->canSerialize())
+                    bool canRemoteTranslate = publishedFormat->queryTypeInfo()->canInterpret() && publishedFormat->queryTypeInfo()->canSerialize() &&
+                                              projectedFormat->queryTypeInfo()->canInterpret() && projectedFormat->queryTypeInfo()->canSerialize();
+                    if (!canRemoteTranslate)
                         throw MakeActivityException(&activity, 0, "CFetchRemoteLookupHandler - translation required, but formats unserializable");
                     msg.append(static_cast<std::underlying_type<RecordTranslationMode>::type>(translationMode));
                     msg.append(publishedFormatCrc);