Merge branch 'HPCC-9090-improvement'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecllibrary/std/Str.ecl

@@ -2,8 +2,16 @@
 ## HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.  All rights reserved.
 ############################################################################## */
 
+
+externals := 
+    SERVICE
+string EncodeBase64(const data src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlBase64Encode';
+data DecodeBase64(const string src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlBase64Decode';
+    END;
+
 EXPORT Str := MODULE
 
+
 /*
   Since this is primarily a wrapper for a plugin, all the definitions for this standard library
   module are included in a single file.  Generally I would expect them in individual files.
@@ -391,4 +399,30 @@ EXPORT string ToHexPairs(data value) := lib_stringlib.StringLib.Data2String(valu
 
 EXPORT data FromHexPairs(string hex_pairs) := lib_stringlib.StringLib.String2Data(hex_pairs);
 
+/*
+ * Encode binary data to base64 string.
+ *
+ * Every 3 data bytes are encoded to 4 base64 characters. If the length of the input is not divisible 
+ * by 3, up to 2 '=' characters are appended to the output. 
+ *
+ *
+ * @param value         The binary data array to process.
+ * @return              Base 64 encoded string.
+ */
+
+EXPORT STRING EncodeBase64(data value) := externals.EncodeBase64(value);
+
+/*
+ * Decode base64 encoded string to binary data.
+ *
+ * If the input is not valid base64 encoding (invalid characters, or ends mid-quartet), an empty
+ * result is returned. Whitespace in the input is skipped.
+ *
+ *
+ * @param value        The base 64 encoded string.
+ * @return             Decoded binary data if the input is valid else zero length data.
+ */
+
+EXPORT DATA DecodeBase64(STRING value) := externals.DecodeBase64(value);
+
 END;

ecllibrary/teststd/str/TestBase64Codec.ecl

@@ -0,0 +1,79 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.  All rights reserved.
+############################################################################## */
+IMPORT Std.Str;
+
+EXPORT TestBase64Codec := MODULE
+
+  EXPORT TestConst := MODULE
+    EXPORT Test01 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'ca')) = x'ca');
+    EXPORT Test02 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafe')) = x'cafe');
+    EXPORT Test03 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafeba')) = x'cafeba');
+    EXPORT Test04 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafebabe')) = x'cafebabe');
+    EXPORT Test05 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafebabeca')) = x'cafebabeca');
+    EXPORT Test06 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafebabecafe')) = x'cafebabecafe');
+    
+    EXPORT Test07 := ASSERT(Str.EncodeBase64(x'ca') = 'yg==');
+    EXPORT Test08 := ASSERT(Str.EncodeBase64(x'cafe') = 'yv4=');
+    EXPORT Test09 := ASSERT(Str.EncodeBase64(x'cafeba') = 'yv66');
+    EXPORT Test10 := ASSERT(Str.EncodeBase64(x'cafebabe') = 'yv66vg==');
+    EXPORT Test11 := ASSERT(Str.EncodeBase64(x'cafebabeca') = 'yv66vso=');
+    EXPORT Test12 := ASSERT(Str.EncodeBase64(x'cafebabecafe') = 'yv66vsr+');
+
+    EXPORT Test13 := ASSERT(Str.DecodeBase64('yg==') = x'ca');
+    EXPORT Test14 := ASSERT(Str.DecodeBase64('yv4=') = x'cafe');
+    EXPORT Test15 := ASSERT(Str.DecodeBase64('yv66') = x'cafeba');
+    EXPORT Test16 := ASSERT(Str.DecodeBase64('yv66vg==') = x'cafebabe');
+    EXPORT Test17 := ASSERT(Str.DecodeBase64('yv66vso=') = x'cafebabeca');
+    EXPORT Test18 := ASSERT(Str.DecodeBase64('yv66vsr+') = x'cafebabecafe');
+
+    /* Invalid printable character e.g.'@' or '#' in the encoded string */	
+    EXPORT Test19 := ASSERT(Str.DecodeBase64('yg@=') = d'') ;
+    EXPORT Test20 := ASSERT(Str.DecodeBase64('#yg=') = d'') ;
+    EXPORT Test21 := ASSERT(Str.DecodeBase64('y#g=') = d'') ;
+    EXPORT Test22 := ASSERT(Str.DecodeBase64('yg#=') = d'') ;
+    EXPORT Test23 := ASSERT(Str.DecodeBase64('yg=#') = d'') ;
+
+    /* Invalid non-printable character e.g.'\t' or '\b' in the encoded string */
+    EXPORT Test24 := ASSERT(Str.DecodeBase64('y'+x'09'+'g=') = d'');
+    EXPORT Test25 := ASSERT(Str.DecodeBase64('y'+x'08'+'g=') = d'');
+
+    /* Missing pad character from encoded string (length error) */	
+    EXPORT Test26 := ASSERT(Str.DecodeBase64('yg=') = d'') ;
+    EXPORT Test27 := ASSERT(Str.DecodeBase64('yv66vg=') = d'');
+
+    /* Length error in encoded string. It doesn't multiple of 4 */	
+    EXPORT Test28 := ASSERT(Str.DecodeBase64('yv66v==') = d'') ;
+    EXPORT Test29 := ASSERT(Str.DecodeBase64('yv66vg=') = d'') ;
+
+    /* Space(s) in the encoded string */	
+    EXPORT Test30 := ASSERT(Str.DecodeBase64('y g==') = x'ca') ;
+    EXPORT Test31 := ASSERT(Str.DecodeBase64('y g = = ') = x'ca') ;
+	
+    /* Long text encoding. Encoder inserts '\n' to break long output lines. */
+    EXPORT DATA text := 
+           d'Man is distinguished, not only by his reason, but by this singular passion from other animals, '
+         + d'which is a lust of the mind, that by a perseverance of delight in the continued and '
+         + d'indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure.';
+
+    EXPORT STRING encodedText :=
+           'TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0'+'\n'
+         + 'aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1'+'\n'
+         + 'c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0'+'\n'
+         + 'aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdl'+'\n'
+         + 'LCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=';
+
+    EXPORT TEST32 := ASSERT(Str.EncodeBase64(text) = encodedText);
+
+    /* Long encoded text decoding. Decoder skips '\n' characters in input string. */
+    EXPORT DATA decodedText := Str.DecodeBase64(encodedText);
+    EXPORT TEST33 := ASSERT(decodedText = text);
+
+    /* Test encoder for zero length and full zero data input string. */
+    EXPORT TEST34 := ASSERT(Str.DecodeBase64('') = d'');
+    EXPORT TEST35 := ASSERT(Str.DecodeBase64(''+x'00000000') = d'');
+  END;
+
+  EXPORT Main := [EVALUATE(TestConst)]; 
+END;
+

rtl/eclrtl/eclrtl.cpp

@@ -5621,6 +5621,41 @@ IAtom * rtlCreateFieldNameAtom(const char * name)
     return createAtom(name);
 }
 
+void rtlBase64Encode(size32_t & tlen, char * & tgt, size32_t slen, const void * src)
+{
+    tlen = 0;
+    tgt = NULL;
+    if (slen)
+    {
+        StringBuffer out;
+        JBASE64_Encode(src, slen, out);
+        tlen = out.length();
+        if (tlen)
+        {
+            char * data  = (char *) rtlMalloc(tlen);
+            out.getChars(0, tlen, data);
+            tgt = data;
+        }
+    }
+}
+
+void rtlBase64Decode(size32_t & tlen, void * & tgt, size32_t slen, const char * src)
+{
+    tlen = 0;
+    if (slen)
+    {
+        StringBuffer out;
+        if (JBASE64_Decode(slen, src, out))
+            tlen = out.length();
+        if (tlen)
+        {
+            char * data = (char *) rtlMalloc(tlen);
+            out.getChars(0, tlen, data);
+            tgt = (void *) data;
+        }
+    }
+}
+
 
 //---------------------------------------------------------------------------
 

rtl/eclrtl/eclrtl.hpp

@@ -722,6 +722,31 @@ ECLRTL_API void rtlFreeException(IException * e);
 
 ECLRTL_API IAtom * rtlCreateFieldNameAtom(const char * name);
 
+/**
+ * Wrapper function to encode input binary data with base 64 code.
+ *
+ * @param tlen          Encoded string length
+ * @param tgt           Pointer to encoded string
+ * @param slen          Input binary data length
+ * @param src           Pointer to input binary data
+ * @see                 void JBASE64_Encode(const void *data, long length, StringBuffer &out, bool addLineBreaks=true)
+ *                      function in jutil library
+ */
+ECLRTL_API void rtlBase64Encode(size32_t & tlen, char * & tgt, size32_t slen, const void * src);
+
+/**
+ * Wrapper function to decode base 64 encoded string.
+ * It handles when the decoder fails to decode string.
+ *
+ * @param tlen          Decoded data length
+ * @param tgt           Pointer to decoded data
+ * @param slen          Input string length
+ * @param src           Pointer to input string
+ * @see                 bool JBASE64_Decode(const char *in, long length, StringBuffer &out) function
+ *                      in jutil library.
+ */
+ECLRTL_API void rtlBase64Decode(size32_t & tlen, void * & tgt, size32_t slen, const char * src);
+
 //Test functions:
 ECLRTL_API void rtlTestGetPrimes(size32_t & len, void * & data);
 ECLRTL_API void rtlTestFibList(bool & outAll, size32_t & outSize, void * & outData, bool inAll, size32_t inSize, const void * inData);

system/jlib/jutil.cpp

@@ -1146,6 +1146,62 @@ MemoryBuffer &JBASE64_Decode(const char *incs, MemoryBuffer &out)
 }
 
 
+
+bool JBASE64_Decode(size32_t length, const char *incs, StringBuffer &out)
+{
+    out.ensureCapacity(((length / 4) + 1) * 3);
+
+    const char * end = incs + length;
+    unsigned char c1;
+    unsigned char c[4];
+    unsigned cIndex = 0;
+    unsigned char d1, d2, d3, d4;
+    bool fullQuartetDecoded = false;
+
+    while (incs < end)
+    {
+        c1 = *incs++;
+
+        if (isspace(c1))
+            continue;
+
+        if (!BASE64_dec[c1] && ('A' != c1) && (pad != c1))
+        {
+            // Forbidden char
+            fullQuartetDecoded = false;
+            break;
+        }
+
+        c[cIndex++] = c1;
+        fullQuartetDecoded = false;
+
+        if (4 == cIndex)
+        {
+            d1 = BASE64_dec[c[0]];
+            d2 = BASE64_dec[c[1]];
+            d3 = BASE64_dec[c[2]];
+            d4 = BASE64_dec[c[3]];
+
+            out.append((char)((d1 << 2) | (d2 >> 4)));
+            fullQuartetDecoded = true;
+
+            if (pad == c[2])
+                break;
+
+            out.append((char)((d2 << 4) | (d3 >> 2)));
+
+            if( pad == c[3])
+                break;
+
+            out.append((char)((d3 << 6) | d4));
+
+            cIndex = 0;
+        }
+    }
+
+    return fullQuartetDecoded;
+}
+
 static inline void encode5_32(const byte *in,StringBuffer &out)
 {
     // 5 bytes in 8 out

system/jlib/jutil.hpp

@@ -130,6 +130,18 @@ extern jlib_decl MemoryBuffer &JBASE64_Decode(const char *in, MemoryBuffer &out)
 extern jlib_decl StringBuffer &JBASE64_Decode(ISimpleReadStream &in, StringBuffer &out);
 extern jlib_decl MemoryBuffer &JBASE64_Decode(ISimpleReadStream &in, MemoryBuffer &out);
 
+/**
+ * Decode base 64 encoded string.
+ * It handles forbidden printable and non-printable chars. Space(s) inserted among the valid chars,
+ * missing pad chars and invalid length.
+ *
+ * @param length        Length of the input string.
+ * @param in            Pointer to base64 encoded string
+ * @param out           Decoded string if the input is valid
+ * @return              True when success
+ */
+extern jlib_decl bool JBASE64_Decode(size32_t length, const char *in, StringBuffer &out);
+
 extern jlib_decl void JBASE32_Encode(const char *in,StringBuffer &out);  // result all lower
 extern jlib_decl void JBASE32_Decode(const char *in,StringBuffer &out);