HPCC-9090 Add base 64 encode/decode functions to the ecllibrary

This new feature contains these implementations:

1. Add a new implementation of JBASE64_Decode which takes a pointer to input
   string and the length of it. It skips space and new line characters.
   Validate characters in input string. If invalid one detected this function
   returns with false.

2. Add wrapper functions rtlBase64Encode() and rtlBase64Decode() into eclrtl
   (.hpp and .cpp) to call Base 64 encode/decode functions (implemented in
   jutil.cpp).
   If base 64 decode returns with false (decoding fails) then rtlBase64Decode()
   returns with zero length data.

3. Add external references and exports for functions implemented in 2. into
   Str.ecl.

4. Add test code TestBase64Codec.ecl into ecllibrary/teststd/str/ directory.

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>
Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecllibrary/std/Str.ecl

@@ -2,8 +2,16 @@
 ## HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.  All rights reserved.
 ############################################################################## */
 
+
+externals := 
+    SERVICE
+string EncodeBase64(const data src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlBase64Encode';
+data DecodeBase64(const string src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlBase64Decode';
+    END;
+
 EXPORT Str := MODULE
 
+
 /*
   Since this is primarily a wrapper for a plugin, all the definitions for this standard library
   module are included in a single file.  Generally I would expect them in individual files.
@@ -391,4 +399,30 @@ EXPORT string ToHexPairs(data value) := lib_stringlib.StringLib.Data2String(valu
 
 EXPORT data FromHexPairs(string hex_pairs) := lib_stringlib.StringLib.String2Data(hex_pairs);
 
+/*
+ * Encode binary data to base64 string.
+ *
+ * Every 3 data bytes are encoded to 4 base64 characters. If the length of the input is not divisible 
+ * by 3, up to 2 '=' characters are appended to the output. 
+ *
+ *
+ * @param value         The binary data array to process.
+ * @return              Base 64 encoded string.
+ */
+
+EXPORT STRING EncodeBase64(data value) := externals.EncodeBase64(value);
+
+/*
+ * Decode base64 encoded string to binary data.
+ *
+ * If the input is not valid base64 encoding (invalid characters, or ends mid-quartet), an empty
+ * result is returned. Whitespace in the input is skipped.
+ *
+ *
+ * @param value        The base 64 encoded string.
+ * @return             Decoded binary data if the input is valid else zero length data.
+ */
+
+EXPORT DATA DecodeBase64(STRING value) := externals.DecodeBase64(value);
+
 END;

ecllibrary/teststd/str/TestBase64Codec.ecl

@@ -0,0 +1,79 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.  All rights reserved.
+############################################################################## */
+IMPORT Std.Str;
+
+EXPORT TestBase64Codec := MODULE
+
+  EXPORT TestConst := MODULE
+    EXPORT Test01 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'ca')) = x'ca');
+    EXPORT Test02 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafe')) = x'cafe');
+    EXPORT Test03 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafeba')) = x'cafeba');
+    EXPORT Test04 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafebabe')) = x'cafebabe');
+    EXPORT Test05 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafebabeca')) = x'cafebabeca');
+    EXPORT Test06 := ASSERT(Str.DecodeBase64(Str.EncodeBase64(x'cafebabecafe')) = x'cafebabecafe');
+    
+    EXPORT Test07 := ASSERT(Str.EncodeBase64(x'ca') = 'yg==');
+    EXPORT Test08 := ASSERT(Str.EncodeBase64(x'cafe') = 'yv4=');
+    EXPORT Test09 := ASSERT(Str.EncodeBase64(x'cafeba') = 'yv66');
+    EXPORT Test10 := ASSERT(Str.EncodeBase64(x'cafebabe') = 'yv66vg==');
+    EXPORT Test11 := ASSERT(Str.EncodeBase64(x'cafebabeca') = 'yv66vso=');
+    EXPORT Test12 := ASSERT(Str.EncodeBase64(x'cafebabecafe') = 'yv66vsr+');
+
+    EXPORT Test13 := ASSERT(Str.DecodeBase64('yg==') = x'ca');
+    EXPORT Test14 := ASSERT(Str.DecodeBase64('yv4=') = x'cafe');
+    EXPORT Test15 := ASSERT(Str.DecodeBase64('yv66') = x'cafeba');
+    EXPORT Test16 := ASSERT(Str.DecodeBase64('yv66vg==') = x'cafebabe');
+    EXPORT Test17 := ASSERT(Str.DecodeBase64('yv66vso=') = x'cafebabeca');
+    EXPORT Test18 := ASSERT(Str.DecodeBase64('yv66vsr+') = x'cafebabecafe');
+
+    /* Invalid printable character e.g.'@' or '#' in the encoded string */	
+    EXPORT Test19 := ASSERT(Str.DecodeBase64('yg@=') = d'') ;
+    EXPORT Test20 := ASSERT(Str.DecodeBase64('#yg=') = d'') ;
+    EXPORT Test21 := ASSERT(Str.DecodeBase64('y#g=') = d'') ;
+    EXPORT Test22 := ASSERT(Str.DecodeBase64('yg#=') = d'') ;
+    EXPORT Test23 := ASSERT(Str.DecodeBase64('yg=#') = d'') ;
+
+    /* Invalid non-printable character e.g.'\t' or '\b' in the encoded string */
+    EXPORT Test24 := ASSERT(Str.DecodeBase64('y'+x'09'+'g=') = d'');
+    EXPORT Test25 := ASSERT(Str.DecodeBase64('y'+x'08'+'g=') = d'');
+
+    /* Missing pad character from encoded string (length error) */	
+    EXPORT Test26 := ASSERT(Str.DecodeBase64('yg=') = d'') ;
+    EXPORT Test27 := ASSERT(Str.DecodeBase64('yv66vg=') = d'');
+
+    /* Length error in encoded string. It doesn't multiple of 4 */	
+    EXPORT Test28 := ASSERT(Str.DecodeBase64('yv66v==') = d'') ;
+    EXPORT Test29 := ASSERT(Str.DecodeBase64('yv66vg=') = d'') ;
+
+    /* Space(s) in the encoded string */	
+    EXPORT Test30 := ASSERT(Str.DecodeBase64('y g==') = x'ca') ;
+    EXPORT Test31 := ASSERT(Str.DecodeBase64('y g = = ') = x'ca') ;
+	
+    /* Long text encoding. Encoder inserts '\n' to break long output lines. */
+    EXPORT DATA text := 
+           d'Man is distinguished, not only by his reason, but by this singular passion from other animals, '
+         + d'which is a lust of the mind, that by a perseverance of delight in the continued and '
+         + d'indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure.';
+
+    EXPORT STRING encodedText :=
+           'TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0'+'\n'
+         + 'aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1'+'\n'
+         + 'c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0'+'\n'
+         + 'aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdl'+'\n'
+         + 'LCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=';
+
+    EXPORT TEST32 := ASSERT(Str.EncodeBase64(text) = encodedText);
+
+    /* Long encoded text decoding. Decoder skips '\n' characters in input string. */
+    EXPORT DATA decodedText := Str.DecodeBase64(encodedText);
+    EXPORT TEST33 := ASSERT(decodedText = text);
+
+    /* Test encoder for zero length and full zero data input string. */
+    EXPORT TEST34 := ASSERT(Str.DecodeBase64('') = d'');
+    EXPORT TEST35 := ASSERT(Str.DecodeBase64(''+x'00000000') = d'');
+  END;
+
+  EXPORT Main := [EVALUATE(TestConst)]; 
+END;
+

rtl/eclrtl/eclrtl.cpp

@@ -5603,6 +5603,41 @@ IAtom * rtlCreateFieldNameAtom(const char * name)
     return createAtom(name);
 }
 
+void rtlBase64Encode(size32_t & tlen, char * & tgt, size32_t slen, const void * src)
+{
+    tlen = 0;
+    tgt = NULL;
+    if (slen)
+    {
+        StringBuffer out;
+        JBASE64_Encode(src, slen, out);
+        tlen = out.length();
+        if (tlen)
+        {
+            char * data  = (char *) rtlMalloc(tlen);
+            out.getChars(0, tlen, data);
+            tgt = data;
+        }
+    }
+}
+
+void rtlBase64Decode(size32_t & tlen, void * & tgt, size32_t slen, const char * src)
+{
+    tlen = 0;
+    if (slen)
+    {
+        StringBuffer out;
+        if (JBASE64_Decode(slen, src, out))
+            tlen = out.length();
+        if (tlen)
+        {
+            char * data = (char *) rtlMalloc(tlen);
+            out.getChars(0, tlen, data);
+            tgt = (void *) data;
+        }
+    }
+}
+
 
 //---------------------------------------------------------------------------
 

rtl/eclrtl/eclrtl.hpp

@@ -722,6 +722,31 @@ ECLRTL_API void rtlFreeException(IException * e);
 
 ECLRTL_API IAtom * rtlCreateFieldNameAtom(const char * name);
 
+/**
+ * Wrapper function to encode input binary data with base 64 code.
+ *
+ * @param tlen          Encoded string length
+ * @param tgt           Pointer to encoded string
+ * @param slen          Input binary data length
+ * @param src           Pointer to input binary data
+ * @see                 void JBASE64_Encode(const void *data, long length, StringBuffer &out, bool addLineBreaks=true)
+ *                      function in jutil library
+ */
+ECLRTL_API void rtlBase64Encode(size32_t & tlen, char * & tgt, size32_t slen, const void * src);
+
+/**
+ * Wrapper function to decode base 64 encoded string.
+ * It handles when the decoder fails to decode string.
+ *
+ * @param tlen          Decoded data length
+ * @param tgt           Pointer to decoded data
+ * @param slen          Input string length
+ * @param src           Pointer to input string
+ * @see                 bool JBASE64_Decode(const char *in, long length, StringBuffer &out) function
+ *                      in jutil library.
+ */
+ECLRTL_API void rtlBase64Decode(size32_t & tlen, void * & tgt, size32_t slen, const char * src);
+
 //Test functions:
 ECLRTL_API void rtlTestGetPrimes(size32_t & len, void * & data);
 ECLRTL_API void rtlTestFibList(bool & outAll, size32_t & outSize, void * & outData, bool inAll, size32_t inSize, const void * inData);

system/jlib/jutil.cpp

@@ -1146,6 +1146,62 @@ MemoryBuffer &JBASE64_Decode(const char *incs, MemoryBuffer &out)
 }
 
 
+
+bool JBASE64_Decode(size32_t length, const char *incs, StringBuffer &out)
+{
+    out.ensureCapacity(((length / 4) + 1) * 3);
+
+    const char * end = incs + length;
+    unsigned char c1;
+    unsigned char c[4];
+    unsigned cIndex = 0;
+    unsigned char d1, d2, d3, d4;
+    bool fullQuartetDecoded = false;
+
+    while (incs < end)
+    {
+        c1 = *incs++;
+
+        if (isspace(c1))
+            continue;
+
+        if (!BASE64_dec[c1] && ('A' != c1) && (pad != c1))
+        {
+            // Forbidden char
+            fullQuartetDecoded = false;
+            break;
+        }
+
+        c[cIndex++] = c1;
+        fullQuartetDecoded = false;
+
+        if (4 == cIndex)
+        {
+            d1 = BASE64_dec[c[0]];
+            d2 = BASE64_dec[c[1]];
+            d3 = BASE64_dec[c[2]];
+            d4 = BASE64_dec[c[3]];
+
+            out.append((char)((d1 << 2) | (d2 >> 4)));
+            fullQuartetDecoded = true;
+
+            if (pad == c[2])
+                break;
+
+            out.append((char)((d2 << 4) | (d3 >> 2)));
+
+            if( pad == c[3])
+                break;
+
+            out.append((char)((d3 << 6) | d4));
+
+            cIndex = 0;
+        }
+    }
+
+    return fullQuartetDecoded;
+}
+
 static inline void encode5_32(const byte *in,StringBuffer &out)
 {
     // 5 bytes in 8 out

system/jlib/jutil.hpp

@@ -130,6 +130,18 @@ extern jlib_decl MemoryBuffer &JBASE64_Decode(const char *in, MemoryBuffer &out)
 extern jlib_decl StringBuffer &JBASE64_Decode(ISimpleReadStream &in, StringBuffer &out);
 extern jlib_decl MemoryBuffer &JBASE64_Decode(ISimpleReadStream &in, MemoryBuffer &out);
 
+/**
+ * Decode base 64 encoded string.
+ * It handles forbidden printable and non-printable chars. Space(s) inserted among the valid chars,
+ * missing pad chars and invalid length.
+ *
+ * @param length        Length of the input string.
+ * @param in            Pointer to base64 encoded string
+ * @param out           Decoded string if the input is valid
+ * @return              True when success
+ */
+extern jlib_decl bool JBASE64_Decode(size32_t length, const char *in, StringBuffer &out);
+
 extern jlib_decl void JBASE32_Encode(const char *in,StringBuffer &out);  // result all lower
 extern jlib_decl void JBASE32_Decode(const char *in,StringBuffer &out);