Merge pull request #3479 from wangkx/h7948

HPCC-7948 Add CsvEscape option to EclWatch UI/dfuwu/ftbase

Reviewed-By: Renato Golin <rengolin@hpccsystems.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

dali/dfu/dfuwu.cpp

@@ -1316,7 +1316,7 @@ public:
         return CDFUfileformat::decode(queryProperties()->queryProp("@format"));
     }
 
-    virtual void getCsvOptions(StringBuffer &separate,StringBuffer &terminate,StringBuffer &quote) const
+    virtual void getCsvOptions(StringBuffer &separate,StringBuffer &terminate,StringBuffer &quote,StringBuffer &escape) const
     {
         IPropertyTree *t = queryProperties();
         const char *sep=t->queryProp("@csvSeparate");
@@ -1325,17 +1325,22 @@ public:
         terminate.append(ter?ter:"\\n,\\r\\n");
         const char *quo=t->queryProp("@csvQuote");
         quote.append(quo?quo:"'");
+        const char *esc=t->queryProp("@csvEscape");
+        if (esc && *esc)
+            escape.set(esc);
     }
 
-    void setCsvOptions(const char *separate,const char *terminate,const char *quote)
+    void setCsvOptions(const char *separate,const char *terminate,const char *quote,const char *escape)
     {
         IPropertyTree *t = queryUpdateProperties();
-        if (separate)
+        if (separate && *separate)
             t->setProp("@csvSeparate",separate);
-        if (terminate)
+        if (terminate && *terminate)
             t->setProp("@csvTerminate",terminate);
-        if (quote)
+        if (quote && *quote)
             t->setProp("@csvQuote",quote);
+        if (escape && *escape)
+            t->setProp("@csvEscape",escape);
     }
 
     StringBuffer &getRowTag(StringBuffer &str)const 

dali/dfu/dfuwu.hpp

@@ -220,7 +220,7 @@ interface IConstDFUfileSpec: extends IInterface
     virtual StringBuffer &getPartUrl(unsigned clustnum,unsigned partidx, StringBuffer &url,bool iskey=false) const = 0; // idx 0 based
     virtual RemoteFilename &getPartFilename(unsigned clustnum,unsigned partidx, RemoteFilename &rfn, bool iskey=false) const = 0; // idx 0 based
     virtual IPropertyTree *queryPartProperties(unsigned partidx) const = 0;
-    virtual void getCsvOptions(StringBuffer &separate,StringBuffer &terminate,StringBuffer &quote) const = 0; 
+    virtual void getCsvOptions(StringBuffer &separate,StringBuffer &terminate,StringBuffer &quote,StringBuffer &escape) const = 0;
     virtual StringBuffer &getRowTag(StringBuffer &str)const =0;
     virtual void setForeignDali(const SocketEndpoint &ep)=0; // only used for source of copy (for inter-dali copy)
     virtual bool getForeignDali(SocketEndpoint &ep) const =0;
@@ -258,7 +258,7 @@ interface IDFUfileSpec: extends IConstDFUfileSpec
     virtual void setRecordSize(size32_t size) = 0; // may need to be supplied for non 1-1 splits
     virtual void setMaxRecordSize(size32_t size) = 0; 
     virtual void setFormat(DFUfileformat format) = 0; 
-    virtual void setCsvOptions(const char *separate=NULL,const char *terminate=NULL,const char *quote=NULL) = 0;  // NULL for default
+    virtual void setCsvOptions(const char *separate=NULL,const char *terminate=NULL,const char *quote=NULL,const char *escape=NULL) = 0;  // NULL for default
     virtual void setRowTag(const char *str) = 0;
     virtual void setFromXML(const char *xml) = 0;
     virtual void setCompressed(bool set) = 0;

dali/ft/filecopy.hpp

@@ -62,6 +62,7 @@ public:
     StringAttr          separate;
     StringAttr          quote;
     StringAttr          terminate;
+    StringAttr          escape;
     StringAttr          rowTag;
 };
 UtfReader::UtfFormat getUtfFormatType(FileFormatType type);

dali/ft/ftbase.cpp

@@ -54,6 +54,7 @@
 #define FPcsvSeparate       "@csvSeparate"
 #define FPcsvQuote          "@csvQuote"
 #define FPcsvTerminate      "@csvTerminate"
+#define FPcsvEscape         "@csvEscape"
 #define FProwTag            "@rowTag"
 
 
@@ -226,6 +227,7 @@ void FileFormat::deserialize(MemoryBuffer & in)
         ::deserialize(in, separate);
         ::deserialize(in, quote);
         ::deserialize(in, terminate);
+        ::deserialize(in, escape);
         ::deserialize(in, rowTag);
         break;
     }
@@ -274,6 +276,8 @@ bool FileFormat::restore(IPropertyTree * props)
         separate.set(props->queryProp(FPcsvSeparate));
         quote.set(props->queryProp(FPcsvQuote));
         terminate.set(props->queryProp(FPcsvTerminate));
+        if (props->hasProp(FPcsvEscape))
+            escape.set(props->queryProp(FPcsvEscape));
         if (maxRecordSize == 0)
             throwError(DFTERR_MaxRecordSizeZero);
     }
@@ -305,6 +309,8 @@ bool FileFormat::restore(IPropertyTree * props)
         separate.set(props->queryProp(FPcsvSeparate));
         quote.set(props->queryProp(FPcsvQuote));
         terminate.set(props->queryProp(FPcsvTerminate));
+        if (props->hasProp(FPcsvEscape))
+            escape.set(props->queryProp(FPcsvEscape));
         rowTag.set(props->queryProp(FProwTag));
         if (maxRecordSize == 0)
             throwError(DFTERR_MaxRecordSizeZero);
@@ -349,6 +355,7 @@ void FileFormat::save(IPropertyTree * props)
         if (separate)       props->setProp(FPcsvSeparate, separate);
         if (quote)          props->setProp(FPcsvQuote, quote);
         if (terminate)      props->setProp(FPcsvTerminate, terminate);
+        if (escape)         props->setProp(FPcsvEscape, escape);
         if (rowTag)         props->setProp(FProwTag, rowTag);
         break;
     case FFTrecfmvb:
@@ -379,6 +386,7 @@ void FileFormat::serialize(MemoryBuffer & out) const
         ::serialize(out, separate);
         ::serialize(out, quote);
         ::serialize(out, terminate);
+        ::serialize(out, escape);
         ::serialize(out, rowTag);
         break;
     }
@@ -392,6 +400,7 @@ void FileFormat::set(const FileFormat & src)
     separate.set(src.separate);
     quote.set(src.quote);
     terminate.set(src.terminate);
+    escape.set(src.escape);
     rowTag.set(src.rowTag);
 }
 

esp/eclwatch/ws_XSLT/dfu_wuid.xslt

@@ -271,6 +271,7 @@
         <xsl:apply-templates select="SourceNumParts"/>
         <xsl:apply-templates select="SourceDirectory"/>
         <xsl:apply-templates select="SourceCsvSeparate"/>
+        <xsl:apply-templates select="SourceCsvEscape"/>
         <xsl:apply-templates select="SourceCsvTerminate"/>
         <xsl:apply-templates select="SourceCsvQuote"/>
         <xsl:apply-templates select="DestLogicalName"/>

esp/eclwatch/ws_XSLT/fs_sprayForm.xslt

@@ -507,6 +507,12 @@
                   </td>
                </tr>
                <tr>
+                  <td>Escape:</td>
+                  <td>
+                     <input type="text" id="sourceCsvEscape" name="sourceCsvEscape" size="6" value="" onchange="handleSubmitBtn()" onblur="handleSubmitBtn()"/>
+                  </td>
+               </tr>
+               <tr>
                   <td>Line Terminator:</td>
                   <td>
                      <input type="text" id="sourceCsvTerminate" name="sourceCsvTerminate" size="6" value="{$term}" onchange="handleSubmitBtn()" onblur="handleSubmitBtn()"/>

esp/scm/ws_dfu.ecm

@@ -91,6 +91,7 @@ ESPStruct DFUFileDetail
     string CsvSeparate;
     string CsvQuote;
     string CsvTerminate;
+    [min_ver("1.20")] string CsvEscape;
     string Modified;
     string Ecl;
     bool ZipFile(false);
@@ -624,7 +625,7 @@ ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUSearchDataRespons
 
 //  ===========================================================================
 ESPservice [
-    version("1.19"), default_client_version("1.19"),
+    version("1.20"), default_client_version("1.20"),
     noforms, 
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/scm/ws_fs.ecm

@@ -62,6 +62,7 @@ ESPStruct [nil_remove] DFUWorkunit
     [min_ver("1.04")] string SourceCsvSeparate;
     [min_ver("1.04")] string SourceCsvQuote;
     [min_ver("1.04")] string SourceCsvTerminate;
+    [min_ver("1.05")] string SourceCsvEscape;
 
     string TimeStarted;
     string TimeStopped;
@@ -317,6 +318,7 @@ ESPrequest [nil_remove] SprayVariable
     string sourceCsvSeparate;
     string sourceCsvTerminate;
     string sourceCsvQuote;
+    string sourceCsvEscape;
     string sourceRowTag;
 
     string destGroup;
@@ -584,7 +586,7 @@ ESPresponse [exceptions_inline, nil_remove] DeleteDropZoneFilesResponse
 };
 
 ESPservice [
-    version("1.04"), default_client_version("1.04"),
+    version("1.05"), default_client_version("1.05"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPuses ESPstruct DFUWorkunit;

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1710,6 +1710,7 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, co
             throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",name);
     }
 
+    double version = context.getClientVersion();
     offset_t size=queryDistributedFileSystem().getSize(df), recordSize=df->queryAttributes().getPropInt64("@recordSize",0);
 
     CDateTime dt;
@@ -1828,6 +1829,10 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, co
    //@csvTerminate - characters used to terminate a record in a csv.utf file
     FileDetails.setCsvTerminate(df->queryAttributes().queryProp("@csvTerminate"));
 
+   //@csvEscape - character used to define escape for a csv/utf file.
+    if (version > 1.19)
+        FileDetails.setCsvEscape(df->queryAttributes().queryProp("@csvEscape"));
+
   
     //Time and date of the file
     tmpstr.clear();
@@ -1849,7 +1854,6 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, co
         clusterStr.append(cluster);
     }
 
-    double version = context.getClientVersion();
     if (clusterStr.length() > 0)
     {
         FileDetails.setCluster(clusterStr.str());

esp/services/ws_fs/ws_fsService.cpp

@@ -348,14 +348,16 @@ static void DeepAssign(IEspContext &context, IConstDFUWorkUnit *src, IEspDFUWork
 
         if (version > 1.03 && (file->getFormat() == DFUff_csv))
         {
-            StringBuffer separate, terminate, quote;
-            file->getCsvOptions(separate,terminate,quote);
+            StringBuffer separate, terminate, quote, escape;
+            file->getCsvOptions(separate,terminate,quote, escape);
             if(separate.length() > 0)
                 dest.setSourceCsvSeparate(separate.str());
             if(terminate.length() > 0)
                 dest.setSourceCsvTerminate(terminate.str());
             if(quote.length() > 0)
                 dest.setSourceCsvQuote(quote.str());
+            if((version > 1.04) && (escape.length() > 0))
+                dest.setSourceCsvEscape(escape.str());
         }
     }
 
@@ -2029,7 +2031,7 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
             const char* cq = req.getSourceCsvQuote();
             if(cq== NULL)
                 cq = "'";
-            source->setCsvOptions(cs, ct, cq);
+            source->setCsvOptions(cs, ct, cq, req.getSourceCsvEscape());
         }
 
         destination->setLogicalName(destname);