From 4be9e9e7fe54ae2fe6f63d39a8209b41e878196d Mon Sep 17 00:00:00 2001 From: Petri Lehtinen Date: Thu, 17 Feb 2011 10:10:53 +0200 Subject: [PATCH] Add custom memory allocation Thanks to Basile Starynkevitch for the suggestion and initial patch. Thanks to Jonathan Landis and Deron Meranda for showing how this can be utilized for implementing secure memory operations. --- doc/apiref.rst | 72 +++++++++++++++++++++++++ src/Makefile.am | 1 + src/dump.c | 12 ++--- src/hashtable.c | 18 +++---- src/jansson.h | 8 +++ src/jansson_private.h | 5 ++ src/load.c | 16 +++--- src/memory.c | 51 ++++++++++++++++++ src/strbuffer.c | 17 ++++-- src/value.c | 51 +++++++++--------- test/.gitignore | 7 +-- test/suites/api/Makefile.am | 8 +-- test/suites/api/check-exports | 1 + test/suites/api/test_memory_funcs.c | 84 +++++++++++++++++++++++++++++ 14 files changed, 293 insertions(+), 58 deletions(-) create mode 100644 src/memory.c create mode 100644 test/suites/api/test_memory_funcs.c diff --git a/doc/apiref.rst b/doc/apiref.rst index adfd4d6..b1d5c07 100644 --- a/doc/apiref.rst +++ b/doc/apiref.rst @@ -1084,3 +1084,75 @@ copied in a recursive fashion. .. refcounting:: new Returns a deep copy of *value*, or *NULL* on error. + + +Custom memory allocation +======================== + +By default, Jansson uses :func:`malloc()` and :func:`free()` for +memory allocation. These functions can be overridden if custom +behavior is needed. + +.. type:: json_malloc_t + + A typedef for a function pointer with :func:`malloc()`'s + signature:: + + typedef void *(*json_malloc_t)(size_t); + +.. type:: json_free_t + + A typedef for a function pointer with :func:`free()`'s + signature:: + + typedef void (*json_free_t)(void *); + +.. function:: void json_set_alloc_funcs(json_malloc_t malloc_fn, json_free_t free_fn) + + Use *malloc_fn* instead of :func:`malloc()` and *free_fn* instead + of :func:`free()`. This function has to be called before any other + Jansson's API functions to ensure that all memory operations use + the same functions. + +Examples: + +Use the `Boehm's conservative garbage collector`_ for memory +operations:: + + json_set_alloc_funcs(GC_malloc, GC_free); + +.. _Boehm's conservative garbage collector: http://www.hpl.hp.com/personal/Hans_Boehm/gc/ + +Allow storing sensitive data (e.g. passwords or encryption keys) in +JSON structures by zeroing all memory when freed:: + + static void *secure_malloc(size_t size) + { + /* Store the memory area size in the beginning of the block */ + void *ptr = malloc(size + 8); + *((size_t *)ptr) = size; + return ptr + 8; + } + + static void secure_free(void *ptr) + { + size_t size; + + ptr -= 8; + size = *((size_t *)ptr); + + guaranteed_memset(ptr, 0, size); + free(ptr); + } + + int main() + { + json_set_alloc_funcs(secure_malloc, secure_free); + /* ... */ + } + +For more information about the issues of storing sensitive data in +memory, see +http://www.dwheeler.com/secure-programs/Secure-Programs-HOWTO/protect-secrets.html. +The page also examplains the :func:`guaranteed_memset()` function used +in the example and gives a sample implementation for it. diff --git a/src/Makefile.am b/src/Makefile.am index 20483b6..f14bbaa 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,6 +8,7 @@ libjansson_la_SOURCES = \ hashtable.h \ jansson_private.h \ load.c \ + memory.c \ pack_unpack.c \ strbuffer.c \ strbuffer.h \ diff --git a/src/dump.c b/src/dump.c index 0ffdfc6..9c01366 100644 --- a/src/dump.c +++ b/src/dump.c @@ -313,7 +313,7 @@ static int do_dump(const json_t *json, size_t flags, int depth, int (*cmp_func)(const void *, const void *); size = json_object_size(json); - keys = malloc(size * sizeof(object_key_t *)); + keys = jsonp_malloc(size * sizeof(object_key_t *)); if(!keys) goto object_error; @@ -346,7 +346,7 @@ static int do_dump(const json_t *json, size_t flags, int depth, if(dump(separator, separator_length, data) || do_dump(value, flags, depth + 1, dump, data)) { - free(keys); + jsonp_free(keys); goto object_error; } @@ -355,7 +355,7 @@ static int do_dump(const json_t *json, size_t flags, int depth, if(dump(",", 1, data) || dump_indent(flags, depth + 1, 1, dump, data)) { - free(keys); + jsonp_free(keys); goto object_error; } } @@ -363,13 +363,13 @@ static int do_dump(const json_t *json, size_t flags, int depth, { if(dump_indent(flags, depth, 0, dump, data)) { - free(keys); + jsonp_free(keys); goto object_error; } } } - free(keys); + jsonp_free(keys); } else { @@ -432,7 +432,7 @@ char *json_dumps(const json_t *json, size_t flags) return NULL; } - result = strdup(strbuffer_value(&strbuff)); + result = jsonp_strdup(strbuffer_value(&strbuff)); strbuffer_close(&strbuff); return result; diff --git a/src/hashtable.c b/src/hashtable.c index d66b240..de25c21 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -145,7 +145,7 @@ static void hashtable_do_clear(hashtable_t *hashtable) hashtable->free_key(pair->key); if(hashtable->free_value) hashtable->free_value(pair->value); - free(pair); + jsonp_free(pair); } } @@ -155,12 +155,12 @@ static int hashtable_do_rehash(hashtable_t *hashtable) pair_t *pair; size_t i, index, new_size; - free(hashtable->buckets); + jsonp_free(hashtable->buckets); hashtable->num_buckets++; new_size = num_buckets(hashtable); - hashtable->buckets = malloc(new_size * sizeof(bucket_t)); + hashtable->buckets = jsonp_malloc(new_size * sizeof(bucket_t)); if(!hashtable->buckets) return -1; @@ -187,13 +187,13 @@ static int hashtable_do_rehash(hashtable_t *hashtable) hashtable_t *hashtable_create(key_hash_fn hash_key, key_cmp_fn cmp_keys, free_fn free_key, free_fn free_value) { - hashtable_t *hashtable = malloc(sizeof(hashtable_t)); + hashtable_t *hashtable = jsonp_malloc(sizeof(hashtable_t)); if(!hashtable) return NULL; if(hashtable_init(hashtable, hash_key, cmp_keys, free_key, free_value)) { - free(hashtable); + jsonp_free(hashtable); return NULL; } @@ -203,7 +203,7 @@ hashtable_t *hashtable_create(key_hash_fn hash_key, key_cmp_fn cmp_keys, void hashtable_destroy(hashtable_t *hashtable) { hashtable_close(hashtable); - free(hashtable); + jsonp_free(hashtable); } int hashtable_init(hashtable_t *hashtable, @@ -214,7 +214,7 @@ int hashtable_init(hashtable_t *hashtable, hashtable->size = 0; hashtable->num_buckets = 0; /* index to primes[] */ - hashtable->buckets = malloc(num_buckets(hashtable) * sizeof(bucket_t)); + hashtable->buckets = jsonp_malloc(num_buckets(hashtable) * sizeof(bucket_t)); if(!hashtable->buckets) return -1; @@ -237,7 +237,7 @@ int hashtable_init(hashtable_t *hashtable, void hashtable_close(hashtable_t *hashtable) { hashtable_do_clear(hashtable); - free(hashtable->buckets); + jsonp_free(hashtable->buckets); } int hashtable_set(hashtable_t *hashtable, void *key, void *value) @@ -266,7 +266,7 @@ int hashtable_set(hashtable_t *hashtable, void *key, void *value) } else { - pair = malloc(sizeof(pair_t)); + pair = jsonp_malloc(sizeof(pair_t)); if(!pair) return -1; diff --git a/src/jansson.h b/src/jansson.h index 248176e..6e6e77a 100644 --- a/src/jansson.h +++ b/src/jansson.h @@ -229,6 +229,14 @@ char *json_dumps(const json_t *json, size_t flags); int json_dumpf(const json_t *json, FILE *output, size_t flags); int json_dump_file(const json_t *json, const char *path, size_t flags); + +/* custom memory allocation */ + +typedef void *(*json_malloc_t)(size_t); +typedef void (*json_free_t)(void *); + +void json_set_alloc_funcs(json_malloc_t malloc_fn, json_free_t free_fn); + #ifdef __cplusplus } #endif diff --git a/src/jansson_private.h b/src/jansson_private.h index 8bb8057..66caad7 100644 --- a/src/jansson_private.h +++ b/src/jansson_private.h @@ -73,4 +73,9 @@ void jsonp_error_set(json_error_t *error, int line, int column, void jsonp_error_vset(json_error_t *error, int line, int column, const char *msg, va_list ap); +/* Wrappers for custom memory functions */ +void* jsonp_malloc(size_t size); +void jsonp_free(void *ptr); +char *jsonp_strdup(const char *str); + #endif diff --git a/src/load.c b/src/load.c index 5a48dc3..f4fb3b7 100644 --- a/src/load.c +++ b/src/load.c @@ -300,7 +300,7 @@ static void lex_scan_string(lex_t *lex, json_error_t *error) - two \uXXXX escapes (length 12) forming an UTF-16 surrogate pair are converted to 4 bytes */ - lex->value.string = malloc(lex->saved_text.length + 1); + lex->value.string = jsonp_malloc(lex->saved_text.length + 1); if(!lex->value.string) { /* this is not very nice, since TOKEN_INVALID is returned */ goto out; @@ -390,7 +390,7 @@ static void lex_scan_string(lex_t *lex, json_error_t *error) return; out: - free(lex->value.string); + jsonp_free(lex->value.string); } #if JSON_INTEGER_IS_LONG_LONG @@ -503,7 +503,7 @@ static int lex_scan(lex_t *lex, json_error_t *error) strbuffer_clear(&lex->saved_text); if(lex->token == TOKEN_STRING) { - free(lex->value.string); + jsonp_free(lex->value.string); lex->value.string = NULL; } @@ -595,7 +595,7 @@ static int lex_init(lex_t *lex, get_func get, eof_func eof, void *data) static void lex_close(lex_t *lex) { if(lex->token == TOKEN_STRING) - free(lex->value.string); + jsonp_free(lex->value.string); strbuffer_close(&lex->saved_text); } @@ -629,7 +629,7 @@ static json_t *parse_object(lex_t *lex, json_error_t *error) lex_scan(lex, error); if(lex->token != ':') { - free(key); + jsonp_free(key); error_set(error, lex, "':' expected"); goto error; } @@ -637,18 +637,18 @@ static json_t *parse_object(lex_t *lex, json_error_t *error) lex_scan(lex, error); value = parse_value(lex, error); if(!value) { - free(key); + jsonp_free(key); goto error; } if(json_object_set_nocheck(object, key, value)) { - free(key); + jsonp_free(key); json_decref(value); goto error; } json_decref(value); - free(key); + jsonp_free(key); lex_scan(lex, error); if(lex->token != ',') diff --git a/src/memory.c b/src/memory.c new file mode 100644 index 0000000..127b5ac --- /dev/null +++ b/src/memory.c @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2009-2011 Petri Lehtinen + * Copyright (c) 2011 Basile Starynkevitch + * + * Jansson is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See LICENSE for details. + */ + +#include +#include + +#include +#include "jansson_private.h" + +/* memory function pointers */ +static json_malloc_t do_malloc = malloc; +static json_free_t do_free = free; + +void *jsonp_malloc(size_t size) +{ + if(!size) + return NULL; + + return (*do_malloc)(size); +} + +void jsonp_free(void *ptr) +{ + if(!ptr) + return; + + (*do_free)(ptr); +} + +char *jsonp_strdup(const char *str) +{ + char *new_str; + + new_str = jsonp_malloc(strlen(str) + 1); + if(!new_str) + return NULL; + + strcpy(new_str, str); + return new_str; +} + +void json_set_alloc_funcs(json_malloc_t malloc_fn, json_free_t free_fn) +{ + do_malloc = malloc_fn; + do_free = free_fn; +} diff --git a/src/strbuffer.c b/src/strbuffer.c index 92c0892..758e95e 100644 --- a/src/strbuffer.c +++ b/src/strbuffer.c @@ -68,12 +68,21 @@ int strbuffer_append_bytes(strbuffer_t *strbuff, const char *data, int size) { if(strbuff->length + size >= strbuff->size) { - strbuff->size = max(strbuff->size * STRBUFFER_FACTOR, - strbuff->length + size + 1); + size_t new_size; + char *new_value; - strbuff->value = realloc(strbuff->value, strbuff->size); - if(!strbuff->value) + new_size = max(strbuff->size * STRBUFFER_FACTOR, + strbuff->length + size + 1); + + new_value = jsonp_malloc(new_size); + if(!new_value) return -1; + + memcpy(new_value, strbuff->value, strbuff->length); + + jsonp_free(strbuff->value); + strbuff->value = new_value; + strbuff->size = new_size; } memcpy(strbuff->value + strbuff->length, data, size); diff --git a/src/value.c b/src/value.c index 534624c..123c7c0 100644 --- a/src/value.c +++ b/src/value.c @@ -61,16 +61,16 @@ static void value_decref(void *value) json_t *json_object(void) { - json_object_t *object = malloc(sizeof(json_object_t)); + json_object_t *object = jsonp_malloc(sizeof(json_object_t)); if(!object) return NULL; json_init(&object->json, JSON_OBJECT); if(hashtable_init(&object->hashtable, jsonp_hash_key, jsonp_key_equal, - free, value_decref)) + jsonp_free, value_decref)) { - free(object); + jsonp_free(object); return NULL; } @@ -83,7 +83,7 @@ json_t *json_object(void) static void json_delete_object(json_object_t *object) { hashtable_close(&object->hashtable); - free(object); + jsonp_free(object); } size_t json_object_size(const json_t *json) @@ -126,8 +126,9 @@ int json_object_set_new_nocheck(json_t *json, const char *key, json_t *value) /* offsetof(...) returns the size of object_key_t without the last, flexible member. This way, the correct amount is allocated. */ - k = malloc(offsetof(object_key_t, key) + - strlen(key) + 1); if(!k) return -1; + k = jsonp_malloc(offsetof(object_key_t, key) + strlen(key) + 1); + if(!k) + return -1; k->serial = object->serial++; strcpy(k->key, key); @@ -351,7 +352,7 @@ static json_t *json_object_deep_copy(json_t *object) json_t *json_array(void) { - json_array_t *array = malloc(sizeof(json_array_t)); + json_array_t *array = jsonp_malloc(sizeof(json_array_t)); if(!array) return NULL; json_init(&array->json, JSON_ARRAY); @@ -359,9 +360,9 @@ json_t *json_array(void) array->entries = 0; array->size = 8; - array->table = malloc(array->size * sizeof(json_t *)); + array->table = jsonp_malloc(array->size * sizeof(json_t *)); if(!array->table) { - free(array); + jsonp_free(array); return NULL; } @@ -377,8 +378,8 @@ static void json_delete_array(json_array_t *array) for(i = 0; i < array->entries; i++) json_decref(array->table[i]); - free(array->table); - free(array); + jsonp_free(array->table); + jsonp_free(array); } size_t json_array_size(const json_t *json) @@ -454,7 +455,7 @@ static json_t **json_array_grow(json_array_t *array, old_table = array->table; new_size = max(array->size + amount, array->size * 2); - new_table = malloc(new_size * sizeof(json_t *)); + new_table = jsonp_malloc(new_size * sizeof(json_t *)); if(!new_table) return NULL; @@ -463,7 +464,7 @@ static json_t **json_array_grow(json_array_t *array, if(copy) { array_copy(array->table, 0, old_table, 0, array->entries); - free(old_table); + jsonp_free(old_table); return array->table; } @@ -524,7 +525,7 @@ int json_array_insert_new(json_t *json, size_t index, json_t *value) array_copy(array->table, 0, old_table, 0, index); array_copy(array->table, index + 1, old_table, index, array->entries - index); - free(old_table); + jsonp_free(old_table); } else array_move(array, index + 1, index, array->entries - index); @@ -653,14 +654,14 @@ json_t *json_string_nocheck(const char *value) if(!value) return NULL; - string = malloc(sizeof(json_string_t)); + string = jsonp_malloc(sizeof(json_string_t)); if(!string) return NULL; json_init(&string->json, JSON_STRING); - string->value = strdup(value); + string->value = jsonp_strdup(value); if(!string->value) { - free(string); + jsonp_free(string); return NULL; } @@ -688,12 +689,12 @@ int json_string_set_nocheck(json_t *json, const char *value) char *dup; json_string_t *string; - dup = strdup(value); + dup = jsonp_strdup(value); if(!dup) return -1; string = json_to_string(json); - free(string->value); + jsonp_free(string->value); string->value = dup; return 0; @@ -709,8 +710,8 @@ int json_string_set(json_t *json, const char *value) static void json_delete_string(json_string_t *string) { - free(string->value); - free(string); + jsonp_free(string->value); + jsonp_free(string); } static int json_string_equal(json_t *string1, json_t *string2) @@ -728,7 +729,7 @@ static json_t *json_string_copy(json_t *string) json_t *json_integer(json_int_t value) { - json_integer_t *integer = malloc(sizeof(json_integer_t)); + json_integer_t *integer = jsonp_malloc(sizeof(json_integer_t)); if(!integer) return NULL; json_init(&integer->json, JSON_INTEGER); @@ -757,7 +758,7 @@ int json_integer_set(json_t *json, json_int_t value) static void json_delete_integer(json_integer_t *integer) { - free(integer); + jsonp_free(integer); } static int json_integer_equal(json_t *integer1, json_t *integer2) @@ -775,7 +776,7 @@ static json_t *json_integer_copy(json_t *integer) json_t *json_real(double value) { - json_real_t *real = malloc(sizeof(json_real_t)); + json_real_t *real = jsonp_malloc(sizeof(json_real_t)); if(!real) return NULL; json_init(&real->json, JSON_REAL); @@ -804,7 +805,7 @@ int json_real_set(json_t *json, double value) static void json_delete_real(json_real_t *real) { - free(real); + jsonp_free(real); } static int json_real_equal(json_t *real1, json_t *real2) diff --git a/test/.gitignore b/test/.gitignore index cb88169..7b11ea0 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,13 +1,14 @@ logs bin/json_process suites/api/test_array -suites/api/test_equal suites/api/test_copy +suites/api/test_cpp suites/api/test_dump +suites/api/test_equal suites/api/test_load +suites/api/test_memory_funcs suites/api/test_number suites/api/test_object -suites/api/test_simple -suites/api/test_cpp suites/api/test_pack +suites/api/test_simple suites/api/test_unpack diff --git a/test/suites/api/Makefile.am b/test/suites/api/Makefile.am index 1e4a0bb..0021f93 100644 --- a/test/suites/api/Makefile.am +++ b/test/suites/api/Makefile.am @@ -2,24 +2,26 @@ EXTRA_DIST = run check_PROGRAMS = \ test_array \ - test_equal \ test_copy \ test_dump \ + test_equal \ test_load \ - test_simple \ + test_memory_funcs \ test_number \ test_object \ test_pack \ + test_simple \ test_unpack test_array_SOURCES = test_array.c util.h test_copy_SOURCES = test_copy.c util.h test_dump_SOURCES = test_dump.c util.h test_load_SOURCES = test_load.c util.h -test_simple_SOURCES = test_simple.c util.h +test_memory_funcs_SOURCES = test_memory_funcs.c util.h test_number_SOURCES = test_number.c util.h test_object_SOURCES = test_object.c util.h test_pack_SOURCES = test_pack.c util.h +test_simple_SOURCES = test_simple.c util.h test_unpack_SOURCES = test_unpack.c util.h AM_CPPFLAGS = -I$(top_srcdir)/src diff --git a/test/suites/api/check-exports b/test/suites/api/check-exports index 58643ec..d18c529 100755 --- a/test/suites/api/check-exports +++ b/test/suites/api/check-exports @@ -59,6 +59,7 @@ json_vpack_ex json_unpack json_unpack_ex json_vunpack_ex +json_set_alloc_funcs EOF # The list of functions are not exported in the library because they diff --git a/test/suites/api/test_memory_funcs.c b/test/suites/api/test_memory_funcs.c new file mode 100644 index 0000000..1a6681f --- /dev/null +++ b/test/suites/api/test_memory_funcs.c @@ -0,0 +1,84 @@ +#include +#include + +#include "util.h" + +static int malloc_called = 0; +static int free_called = 0; + +/* helper */ +static void create_and_free_complex_object() +{ + json_t *obj; + + obj = json_pack("{s:i,s:n,s:b,s:b,s:{s:s},s:[i,i,i]", + "foo", 42, + "bar", + "baz", 1, + "qux", 0, + "alice", "bar", "baz", + "bob", 9, 8, 7); + + json_decref(obj); +} + +static void *my_malloc(size_t size) +{ + malloc_called += 1; + return malloc(size); +} + +static void my_free(void *ptr) +{ + free_called += 1; + free(ptr); +} + +static void test_simple() +{ + json_set_alloc_funcs(my_malloc, my_free); + create_and_free_complex_object(); + + if(malloc_called != 27 || free_called != 27) + fail("Custom allocation failed"); +} + + +/* + Test the secure memory functions code given in the API reference + documentation, but by using plain memset instead of + guaranteed_memset(). +*/ + +static void *secure_malloc(size_t size) +{ + /* Store the memory area size in the beginning of the block */ + void *ptr = malloc(size + 8); + *((size_t *)ptr) = size; + return ptr + 8; +} + +static void secure_free(void *ptr) +{ + size_t size; + + ptr -= 8; + size = *((size_t *)ptr); + + /*guaranteed_*/memset(ptr, 0, size); + free(ptr); +} + +static void test_secure_funcs(void) +{ + json_set_alloc_funcs(secure_malloc, secure_free); + create_and_free_complex_object(); +} + +int main() +{ + test_simple(); + test_secure_funcs(); + + return 0; +}