You can also roll your own, which has several benefits:
- Not subject to the somewhat restrictive license of iconv
- No restriction on statically linking to avoid ICU version hell or other dynamic-link headaches
- Avoid the need to link a very large library (such as icu or boost) (which, if statically linked, can add tens of MB of size to what might otherwise be a very small binary)
Note: the below assumes you have installed utf8proc, which is very compact. however, if you prefer, you can simply use its header file and copy the one utf8proc_encode_char() function that this code uses.
utf16le_to_utf8.h:
#ifndef UTF16LE_TO_UTF8_H
#define UTF16LE_TO_UTF8_H
enum utf816le_status {
utf816le_status_ok = 0,
utf816le_status_malformed_utf6le_input,
utf816le_status_memory,
utf816le_status_unencodable,
utf816le_status_buffsize
};
/*
* @return converted string, or NULL on error
* @param str input string
* @param len length (in bytes) of input string
* @param buff optional user-provided output buffer. if not provided, the returned
* converted string must be freed by caller using free()
* @param buffsize length of user-provided buffer, or 0 if no buffer provider
* @param out_len pointer to length of converted output, in bytes
* @param status pointer to status, set to non-zero in case of error
*/
unsigned char *utf16le_to_utf8(const unsigned char *str, size_t len,
unsigned char *buff, size_t buffsize,
size_t *out_len,
enum utf816le_status *status);
#endif
utf16le_to_utf8.c:
#include <stdlib.h>
#include <string.h>
#include <utf8proc.h>
#include "utf16le_to_utf8.h"
#if defined( __FreeBSD__ ) || defined( __OpenBSD__ ) || defined( __NetBSD__ )
# include <sys/endian.h>
#elif defined( BSD ) && ( BSD >= 199103 ) || defined( __APPLE__ ) || \
defined( __CYGWIN32__ ) || defined( __DJGPP__ ) || defined( __osf__ )
# include <machine/endian.h>
#elif defined( __linux__ ) || defined( __GNUC__ ) || defined( __GNU_LIBRARY__ )
# if !defined( __MINGW32__ ) && !defined( _AIX )
# include <endian.h>
# if !defined( __BEOS__ )
# include <byteswap.h>
# endif
# endif
#endif
static inline uint16_t little_endian_16(uint16_t v) {
#if __BYTE_ORDER == __LITTLE_ENDIAN
return v;
#else
return (v << 8) | (v >> 8);
#endif
}
static utf8proc_int32_t utf16le_next_codepoint(uint16_t *text, unsigned int max_bytes,
unsigned int *bytes_read) {
uint16_t c1 = little_endian_16(text[0]);
if (c1 >= 0xd800 && c1 < 0xdc00) {
if(max_bytes < 4) {
*bytes_read = 0;
return 0;
}
*bytes_read = 4;
uint16_t c2 = text[1];
return ((c1 & 0x3ff) << 10) + (c2 & 0x3ff) + 0x10000;
}
if(max_bytes < 2) {
*bytes_read = 0;
return 0;
}
*bytes_read = 2;
return c1;
}
unsigned char *utf16le_to_utf8(const unsigned char *str, size_t len,
unsigned char *buff, size_t buffsize,
size_t *out_len,
enum utf816le_status *status) {
if(!buffsize)
buff = NULL;
if(!buff)
buffsize = 0;
unsigned char *dst = buff;
size_t sizeof_dst = buffsize;
size_t written = 0;
*status = utf816le_status_ok;
unsigned char_len;
for(size_t i = 0; i < len; i+= char_len) {
utf8proc_int32_t codepoint = utf16le_next_codepoint((uint16_t *)(str + i), len - i, &char_len);
if(!char_len) { // error! bad utf
*status = utf816le_status_malformed_utf6le_input;
break;
}
// we need at least 4 bytes to encode to utf8. add 1 for terminal null and 1 for good measure
if(sizeof_dst < written + 6) {
if(buffsize > 0) { // user-provided buffer is too small
*status = utf816le_status_buffsize;
break;
}
size_t new_size = sizeof_dst == 0 ? 64 : sizeof_dst * 2;
unsigned char *new_dst = realloc(dst, new_size);
if(!new_dst) { // out of memory!
*status = utf816le_status_memory;
break;
}
dst = new_dst;
sizeof_dst = new_size;
}
utf8proc_ssize_t want = utf8proc_encode_char(codepoint, dst + written);
if(!want) { // error
*status = utf816le_status_unencodable;
break;
}
written += want;
}
if(*status == utf816le_status_ok) {
*out_len = written;
dst[written] = '\0';
return dst;
}
*out_len = 0;
if(dst != buff)
free(dst);
return NULL;
}
which you can use like so:
...
unsigned char *converted = utf16le_to_utf8(utf16buff, utf16byte_count, NULL, 0, &output_len, &status);
if(!converted || !output_len)
fprintf(stderr, "Error! %i\n", status);
else
fprintf(stdout, "Converted to utf8 with length %zu: %s\n", output_len, converted);
free(converted);
}
}
