From 9de24a8c7316e9affa19240a60ae643e4b1bb87e Mon Sep 17 00:00:00 2001 From: Daniel Barney Date: Tue, 21 Jan 2014 13:10:03 -0700 Subject: [PATCH 1/2] fixed rdb v5 checksum skipping. --- src/rdb_load.erl | 10 ++++-- test/rdb_load_SUITE.erl | 72 +++++++++++++++++++++++++++-------------- 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/rdb_load.erl b/src/rdb_load.erl index b05b927..aa808f4 100644 --- a/src/rdb_load.erl +++ b/src/rdb_load.erl @@ -25,7 +25,7 @@ ,load_file/2 ]). --record(state, {first = true, buffer = <<>>}). +-record(state, {first = true, buffer = <<>>,vsn = undefined}). -include("nsync.hrl"). @@ -52,7 +52,7 @@ packet(State, Data, Callback) when State == undefined orelse State#state.first = lists:member(Vsn, [<<"0001">>,<<"0002">>,<<"0003">>, <<"0004">>,<<"0005">>,<<"0006">>]) orelse exit({error, vsn_not_supported}), - packet(#state{buffer = <<>>, first = false}, Rest2, Callback); + packet(#state{buffer = <<>>, first = false,vsn = binary_to_integer(Vsn)}, Rest2, Callback); {error, eof} -> #state{buffer = Data} end; @@ -63,10 +63,14 @@ packet(State, Data, Callback) when State == undefined orelse State#state.first = #state{buffer = Data} end; -packet(#state{buffer=Buffer}, Data, Callback) -> +packet(#state{buffer=Buffer, vsn = Vsn}, Data, Callback) -> case parse(<>, Callback) of {ok, Rest} -> #state{buffer = Rest, first = false}; + {eof, <<_CRC:8/binary,Rest/binary>>} when Vsn >= 5 -> + {eof, Rest}; + {eof, _} when Vsn >= 5 -> + {error,eof}; {eof, Rest} -> {eof, Rest} end. diff --git a/test/rdb_load_SUITE.erl b/test/rdb_load_SUITE.erl index 4bca2fa..2b6ddcf 100644 --- a/test/rdb_load_SUITE.erl +++ b/test/rdb_load_SUITE.erl @@ -33,33 +33,55 @@ all() -> ,zipmap_that_doesnt_compress ,zipmap_with_big_values]. -dictionary(C) -> rdb_load(dictionary, C). -easily_compressible_string_key(C) -> rdb_load(easily_compressible_string_key, C). -empty_database(C) -> rdb_load(empty_database, C). -hash_as_ziplist(C) -> rdb_load(hash_as_ziplist, C). -integer_keys(C) -> rdb_load(integer_keys, C). -intset_16(C) -> rdb_load(intset_16, C). -intset_32(C) -> rdb_load(intset_32, C). -intset_64(C) -> rdb_load(intset_64, C). -keys_with_expiry(C) -> rdb_load(keys_with_expiry, C). -linkedlist(C) -> rdb_load(linkedlist, C). -multiple_databases(C) -> rdb_load(multiple_databases, C). -parser_filters(C) -> rdb_load(parser_filters, C). -rdb_version_5_with_checksum(C) -> rdb_load(rdb_version_5_with_checksum, C). -regular_set(C) -> rdb_load(regular_set, C). -regular_sorted_set(C) -> rdb_load(regular_sorted_set, C). -sorted_set_as_ziplist(C) -> rdb_load(sorted_set_as_ziplist, C). -uncompressible_string_keys(C) -> rdb_load(uncompressible_string_keys, C). -ziplist_that_compresses_easily(C) -> rdb_load(ziplist_that_compresses_easily, C). -ziplist_that_doesnt_compress(C) -> rdb_load(ziplist_that_doesnt_compress, C). -ziplist_with_integers(C) -> rdb_load(ziplist_with_integers, C). -zipmap_that_compresses_easily(C) -> rdb_load(zipmap_that_compresses_easily, C). -zipmap_that_doesnt_compress(C) -> rdb_load(zipmap_that_doesnt_compress, C). -zipmap_with_big_values(C) -> rdb_load(zipmap_with_big_values, C). +dictionary(C) -> + {eof,<<>>} = rdb_load(dictionary, C). +easily_compressible_string_key(C) -> + {eof,<<>>} = rdb_load(easily_compressible_string_key, C). +empty_database(C) -> + {eof,<<>>} = rdb_load(empty_database, C). +hash_as_ziplist(C) -> + {eof,<<>>} = rdb_load(hash_as_ziplist, C). +integer_keys(C) -> + {eof,<<>>} = rdb_load(integer_keys, C). +intset_16(C) -> + {eof,<<>>} = rdb_load(intset_16, C). +intset_32(C) -> + {eof,<<>>} = rdb_load(intset_32, C). +intset_64(C) -> + {eof,<<>>} = rdb_load(intset_64, C). +keys_with_expiry(C) -> + {eof,<<>>} = rdb_load(keys_with_expiry, C). +linkedlist(C) -> + {eof,<<>>} = rdb_load(linkedlist, C). +multiple_databases(C) -> + {eof,<<>>} = rdb_load(multiple_databases, C). +parser_filters(C) -> + {eof,<<>>} = rdb_load(parser_filters, C). +rdb_version_5_with_checksum(C) -> + {eof,<<>>} = rdb_load(rdb_version_5_with_checksum, C). +regular_set(C) -> + {eof,<<>>} = rdb_load(regular_set, C). +regular_sorted_set(C) -> + {eof,<<>>} = rdb_load(regular_sorted_set, C). +sorted_set_as_ziplist(C) -> + {eof,<<>>} = rdb_load(sorted_set_as_ziplist, C). +uncompressible_string_keys(C) -> + {state,_,_,_} = rdb_load(uncompressible_string_keys, C). +ziplist_that_compresses_easily(C) -> + {eof,<<>>} = rdb_load(ziplist_that_compresses_easily, C). +ziplist_that_doesnt_compress(C) -> + {eof,<<>>} = rdb_load(ziplist_that_doesnt_compress, C). +ziplist_with_integers(C) -> + {eof,<<>>} = rdb_load(ziplist_with_integers, C). +zipmap_that_compresses_easily(C) -> + {eof,<<>>} = rdb_load(zipmap_that_compresses_easily, C). +zipmap_that_doesnt_compress(C) -> + {eof,<<>>} = rdb_load(zipmap_that_doesnt_compress, C). +zipmap_with_big_values(C) -> + {state,_,_,_} = rdb_load(zipmap_with_big_values, C). rdb_load(TestFile, Config) -> FileName = filename:join(?config(data_dir, Config), atom_to_list(TestFile) ++ ".rdb"), rdb_load:load_file(FileName, - fun (_) -> ok end), - ok. + fun (_) -> ok end). From bf3fae09be2f37a3be131df32c8b0a7c3ad02465 Mon Sep 17 00:00:00 2001 From: Daniel Barney Date: Thu, 22 May 2014 09:07:03 -0600 Subject: [PATCH 2/2] fixed large list encoding --- src/rdb_load.erl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/rdb_load.erl b/src/rdb_load.erl index aa808f4..887a03e 100644 --- a/src/rdb_load.erl +++ b/src/rdb_load.erl @@ -204,13 +204,8 @@ rdb_len(<>) -> end; _ -> case Rest of - <> -> - case <> of - <> -> - {ok, false, Val, Rest1}; - _ -> - exit({error, eof}) - end; + <> -> + {ok, false, Val, Rest1}; _ -> exit({error, eof}) end