feat(): add auto opening of column families, streaming, enumerable and collectable

This commit is contained in:
Ryan Schmukler 2017-03-21 14:09:51 -04:00
parent 131b5df024
commit aab14d167e
20 changed files with 263 additions and 601 deletions

View File

@ -6,16 +6,12 @@ defmodule Rox do
alias __MODULE__.{DB,ColumnFamily,Native,Utils,Cursor}
@opts_to_convert_to_bitlists [:db_log_dir, :wal_dir]
@type compaction_style :: :level | :universal | :fifo | :none
@type compression_type :: :snappy | :zlib | :bzip2 | :lz4 | :lz4h | :none
@type key :: String.t | binary
@type value :: any
@type iterator_mode :: :start | :end | {:from, key, :forward | :backward}
@opaque snapshot_handle :: :erocksdb.snapshot_handle
@type file_path :: String.t
@ -41,6 +37,7 @@ defmodule Rox do
@type db_options :: [
{:total_threads, pos_integer} |
{:optimize_level_type_compaction_memtable_memory_budget, integer} |
{:auto_create_column_families, boolean} |
{:create_if_missing, boolean} |
{:max_open_files, pos_integer} |
{:compression_type, compression_type} |
@ -80,15 +77,68 @@ defmodule Rox do
]
@doc """
Open a RocksDB with the specified database options and optional `column_families`.
Open a RocksDB with the optional `db_opts` and `column_families`.
If `column_families` are provided, a 3 element tuple will be returned with
the second element being a map of column family names to `Rox.ColumnFamily` handles.
The column families must have already been created via `create_cf` or the option
`auto_create_column_families` can be set to `true`. If it is, the `db_opts` will be
used to create the column families.
The database will automatically be closed when the BEAM VM releases it for garbage collection.
"""
@spec open(file_path, db_options) :: {:ok, DB.t} | {:error, any}
@spec open(file_path, db_options, [ColumnFamily.name]) ::
{:ok, DB.t} |
{:ok, DB.t, %{ColumnFamily.name => ColumnFamily.t}} |
{:error, any}
def open(path, db_opts \\ [], column_families \\ []) when is_binary(path) and is_list(db_opts) and is_list(column_families) do
with {:ok, result} <- Native.open(path, to_map(db_opts), column_families) do
{:ok, DB.wrap_resource(result)}
db_opts =
to_map(db_opts)
auto_create_cfs? =
db_opts[:auto_create_column_families]
case column_families do
[] ->
do_open_db_with_no_cf(path, db_opts)
_ ->
# First try opening with existing column families
with {:ok, db} <- Native.open(path, db_opts, column_families),
db <- DB.wrap_resource(db),
{:ok, cf_handles} <- map_or_error(column_families, &cf_handle(db, &1)) do
cf_map =
Enum.zip(column_families, cf_handles)
|> Enum.into(%{})
{:ok, db, cf_map}
else
{:error, << "Invalid argument: Column family not found:", _rest :: binary >>} when auto_create_cfs? ->
do_open_db_and_create_cfs(path, db_opts, column_families)
other ->
other
end
end
end
defp do_open_db_with_no_cf(path, opts) do
with {:ok, db} <- Native.open(path, opts, []) do
{:ok, DB.wrap_resource(db)}
end
end
defp do_open_db_and_create_cfs(path, opts, column_families) do
with {:ok, db} <- do_open_db_with_no_cf(path, opts),
{:ok, cf_handles} <- map_or_error(column_families, &create_cf(db, &1, opts)) do
cf_map =
Enum.zip(column_families, cf_handles)
|> Enum.into(%{})
{:ok, db, cf_map}
end
end
@ -96,13 +146,26 @@ defmodule Rox do
Create a column family in `db` with `name` and `opts`.
"""
@spec create_cf(DB.t, String.t, db_options) :: {:ok, ColumnFamily.t} | {:error, any}
@spec create_cf(DB.t, ColumnFamily.name, db_options) :: {:ok, ColumnFamily.t} | {:error, any}
def create_cf(%DB{resource: raw_db} = db, name, opts \\ []) do
with {:ok, result} <- Native.create_cf(raw_db, name, to_map(opts)) do
{:ok, ColumnFamily.wrap_resource(db, result, name)}
end
end
@doc """
Gets an existing `ColumnFamily.t` from the database.
The column family must have been created via `create_cf/2` or from `open/3` with the `auto_create_column_families` option.
"""
@spec cf_handle(DB.t, ColumnFamily.name) :: {:ok, ColumnFamily.t} | {:error, any}
def cf_handle(%DB{resource: raw_db} = db, name) do
with {:ok, result} <- Native.cf_handle(raw_db, name) do
{:ok, ColumnFamily.wrap_resource(db, result, name)}
end
end
@doc """
Put a key/value pair into the specified database or column family.
@ -143,22 +206,22 @@ defmodule Rox do
Returns a `Cursor.t` which will iterate records from the provided database or
column family.
Optionally takes an `iterator_mode`. Defaults to `:start`.
Optionally takes an `Iterator.mode`. Defaults to `:start`.
The default arguments of this function is used for the `Enumerable` implementation
for `DB` and `ColumnFamily` structs.
"""
@spec stream(DB.t | ColumnFamily.t, iterator_mode) :: Cursor.t
@spec stream(DB.t | ColumnFamily.t, Iterator.mode) :: Cursor.t | {:error, any}
def stream(db_or_cf, mode \\ :start)
def stream(%DB{resource: db}, mode) do
with {:ok, resource} = Native.iterate(db, mode) do
{:ok, Cursor.wrap_resource(resource)}
Cursor.wrap_resource(resource, mode)
end
end
def stream(%ColumnFamily{db_resource: db, cf_resource: cf}, mode) do
with {:ok, resource} = Native.iterate_cf(db, cf, mode) do
{:ok, Cursor.wrap_resource(resource)}
Cursor.wrap_resource(resource, mode)
end
end
@ -180,4 +243,24 @@ defmodule Rox do
defp to_map(map) when is_map(map), do: map
defp to_map([]), do: %{}
defp to_map(enum), do: Enum.into(enum, %{})
defp map_or_error(list, fun) do
do_map_or_error(list, fun, [])
end
defp do_map_or_error([], _fun, results), do: {:ok, :lists.reverse(results)}
defp do_map_or_error([item | rest], fun, results) do
case fun.(item) do
{:error, _} = err ->
err
{:ok, result} ->
do_map_or_error(rest, fun, [result | results])
result ->
do_map_or_error(rest, fun, [result | results])
end
end
end

View File

@ -1 +0,0 @@
ryan@amplitude.local.94644

View File

@ -18,6 +18,7 @@ defmodule Rox.ColumnFamily do
}
defstruct [:db_reference, :db_resource, :cf_resource, :name]
@type name :: binary
@doc false
def wrap_resource(%DB{resource: db_resource, reference: db_reference}, resource, name) do
@ -34,4 +35,38 @@ defmodule Rox.ColumnFamily do
"#Rox.ColumnFamily<#{to_doc(handle.db_reference, opts)}>.#{handle.name}"
end
end
defimpl Enumerable do
def count(cf), do: {:ok, Rox.count(cf)}
def member?(cf, {key, val}) do
with {:ok, stored_val} <- Rox.get(cf, key) do
stored_val == {:ok, val}
else
_ -> {:ok, false}
end
end
def member?(_, _), do: {:ok, false}
def reduce(cf, cmd, fun) do
Rox.stream(cf)
|> Enumerable.reduce(cmd, fun)
end
end
defimpl Collectable do
def into(cf) do
collector_fun = fn
cf, {:cont, {key, val}} ->
:ok = Rox.put(cf, key, val)
cf
cf, :done ->
cf
_, :halt ->
:ok
end
{cf, collector_fun}
end
end
end

View File

@ -6,16 +6,51 @@ defmodule Rox.Cursor do
@typedoc "A cursor for iterating over a database or column family"
@type t :: %__MODULE__{
resource: binary
resource: binary, mode: mode
}
defstruct [:resource]
defstruct [:resource, :mode]
@type mode :: :start | :end | {:from, Rox.key, :forward | :backward}
@doc false
def wrap_resource(resource), do: %__MODULE__{resource: resource}
def wrap_resource(resource, mode) do
%__MODULE__{resource: resource, mode: mode}
end
defimpl Inspect do
def inspect(handle, _) do
def inspect(_, _) do
"#Rox.Cursor<>"
end
end
defimpl Enumerable do
alias Rox.{Cursor,Native,Utils}
def count(_), do: {:error, __MODULE__}
def member?(_, _), do: {:error, __MODULE__}
def reduce(%Cursor{resource: raw, mode: mode}, {:halt, acc}, _fun) do
Native.iterator_reset(raw, mode)
{:halted, acc}
end
def reduce(%Cursor{} = cursor, {:suspend, acc}, fun) do
{:suspended, acc, &reduce(cursor, &1, fun)}
end
def reduce(%Cursor{resource: raw, mode: mode} = cursor, {:cont, acc}, fun) do
case Native.iterator_next(raw) do
:done ->
Native.iterator_reset(raw, mode)
{:done, acc}
{key, value} ->
value =
Utils.decode(value)
reduce(cursor, fun.({key, value}, acc), fun)
end
end
end
end

View File

@ -5,7 +5,7 @@ defmodule Rox.DB do
For working with the database, see the functions in the top
level `Rox` module.
Implements the `Collectable` and `Enumerable` protocol.
Implements the `Collectable` and `Enumerable` protocols.
"""
@ -26,4 +26,38 @@ defmodule Rox.DB do
"#Rox.DB<#{to_doc(handle.reference, opts)}>"
end
end
defimpl Enumerable do
def count(db), do: {:ok, Rox.count(db)}
def member?(db, {key, val}) do
with {:ok, stored_val} <- Rox.get(db, key) do
stored_val == {:ok, val}
else
_ -> {:ok, false}
end
end
def member?(_, _), do: {:ok, false}
def reduce(db, cmd, fun) do
Rox.stream(db)
|> Enumerable.reduce(cmd, fun)
end
end
defimpl Collectable do
def into(db) do
collector_fun = fn
db, {:cont, {key, val}} when is_binary(key) ->
:ok = Rox.put(db, key, val)
db
db, :done ->
db
_, :halt ->
:ok
end
{db, collector_fun}
end
end
end

View File

@ -5,6 +5,7 @@ defmodule Rox.Native do
def count(_), do: raise "Nif not loaded"
def count_cf(_, _), do: raise "Nif not loaded"
def create_cf(_, _, _), do: raise "Nif not loaded"
def cf_handle(_, _), do: raise "Nif not loaded"
def put(_, _, _, _), do: raise "Nif not loaded"
def put_cf(_, _, _, _, _), do: raise "Nif not loaded"
def get(_, _, _), do: raise "Nif not loaded"
@ -12,4 +13,5 @@ defmodule Rox.Native do
def iterate(_, _), do: raise "Nif not loaded"
def iterate_cf(_, _, _), do: raise "Nif not loaded"
def iterator_next(_), do: raise "Nif not loaded"
def iterator_reset(_, _), do: raise "Nif not loaded"
end

View File

@ -5,8 +5,8 @@ defmodule Rox.Utils do
inline: [{:decode, 1}, {:encode, 1}]
]
def decode({:ok, << "_$rx", encoded :: binary >>}), do: {:ok, :erlang.binary_to_term(encoded)}
def decode(<< "_$rx", encoded :: binary >>), do: :erlang.binary_to_term(encoded)
def decode({:ok, << "_$rx:", encoded :: binary >>}), do: {:ok, :erlang.binary_to_term(encoded)}
def decode(<< "_$rx:", encoded :: binary >>), do: :erlang.binary_to_term(encoded)
def decode(other), do: other
def encode(val) when is_binary(val), do: val

View File

@ -444,7 +444,7 @@ fn count_cf<'a>(env: NifEnv<'a>, args: &[NifTerm<'a>]) -> NifResult<NifTerm<'a>>
let db_arc: ResourceArc<DBHandle> = args[0].decode()?;
let db_handle = db_arc.deref();
let cf_arc: ResourceArc<CFHandle> = args[0].decode()?;
let cf_arc: ResourceArc<CFHandle> = args[1].decode()?;
let cf = cf_arc.deref().cf;
@ -474,6 +474,18 @@ fn create_cf<'a>(env: NifEnv<'a>, args: &[NifTerm<'a>]) -> NifResult<NifTerm<'a>
Ok(resp)
}
fn cf_handle<'a>(env: NifEnv<'a>, args: &[NifTerm<'a>]) -> NifResult<NifTerm<'a>> {
let db_arc: ResourceArc<DBHandle> = args[0].decode()?;
let db = db_arc.deref().db.read().unwrap();
let name: &str = args[1].decode()?;
match db.cf_handle(name) {
Some(cf) => Ok((atoms::ok(), ResourceArc::new(CFHandle{cf: cf})).encode(env)),
None => Ok((atoms::error(), format!("Could not find ColumnFamily named {}", name)).encode(env))
}
}
fn put<'a>(env: NifEnv<'a>, args: &[NifTerm<'a>]) -> NifResult<NifTerm<'a>> {
let db_arc: ResourceArc<DBHandle> = args[0].decode()?;
let db = db_arc.deref().db.write().unwrap();
@ -622,10 +634,22 @@ fn iterator_next<'a>(env: NifEnv<'a>, args: &[NifTerm<'a>]) -> NifResult<NifTerm
}
}
fn iterator_reset<'a>(env: NifEnv<'a>, args: &[NifTerm<'a>]) -> NifResult<NifTerm<'a>> {
let iter_arc: ResourceArc<IteratorHandle> = args[0].decode()?;
let mut iter = iter_arc.deref().iter.write().unwrap();
let iterator_mode = decode_iterator_mode(args[1])?;
iter.set_mode(iterator_mode);
Ok(atoms::ok().encode(env))
}
rustler_export_nifs!(
"Elixir.Rox.Native",
[("open", 3, open),
("create_cf", 3, create_cf),
("cf_handle", 2, cf_handle),
("put", 4, put),
("put_cf", 5, put_cf),
("count", 1, count),
@ -633,6 +657,7 @@ rustler_export_nifs!(
("iterate", 2, iterate),
("iterate_cf", 3, iterate_cf),
("iterator_next", 1, iterator_next),
("iterator_reset", 2, iterator_reset),
("get", 3, get),
("get_cf", 4, get_cf)],
Some(on_load)

Binary file not shown.

View File

@ -1 +0,0 @@
MANIFEST-000006

View File

@ -1 +0,0 @@
14adbe8d53ad7508-4cfd51f682c1de4e

View File

View File

@ -1,237 +0,0 @@
2017/03/20-20:36:05.926242 b0ea8000 RocksDB version: 4.13.0
2017/03/20-20:36:05.926262 b0ea8000 Git sha rocksdb_build_git_sha:f201a44b4102308b840b15d9b89122af787476f1
2017/03/20-20:36:05.926265 b0ea8000 Compile date Mar 20 2017
2017/03/20-20:36:05.926266 b0ea8000 DB SUMMARY
2017/03/20-20:36:05.926306 b0ea8000 SST files in some_db.rocks dir, Total Num: 0, files:
2017/03/20-20:36:05.926308 b0ea8000 Write Ahead Log file in some_db.rocks:
2017/03/20-20:36:05.926311 b0ea8000 Options.error_if_exists: 0
2017/03/20-20:36:05.926312 b0ea8000 Options.create_if_missing: 1
2017/03/20-20:36:05.926313 b0ea8000 Options.paranoid_checks: 1
2017/03/20-20:36:05.926314 b0ea8000 Options.env: 0x1679fa18
2017/03/20-20:36:05.926315 b0ea8000 Options.info_log: 0x7fd4ff4aa740
2017/03/20-20:36:05.926316 b0ea8000 Options.max_open_files: -1
2017/03/20-20:36:05.926317 b0ea8000 Options.max_file_opening_threads: 16
2017/03/20-20:36:05.926318 b0ea8000 Options.max_total_wal_size: 0
2017/03/20-20:36:05.926319 b0ea8000 Options.disableDataSync: 0
2017/03/20-20:36:05.926320 b0ea8000 Options.use_fsync: 0
2017/03/20-20:36:05.926321 b0ea8000 Options.max_log_file_size: 0
2017/03/20-20:36:05.926322 b0ea8000 Options.max_manifest_file_size: 18446744073709551615
2017/03/20-20:36:05.926323 b0ea8000 Options.log_file_time_to_roll: 0
2017/03/20-20:36:05.926324 b0ea8000 Options.keep_log_file_num: 1000
2017/03/20-20:36:05.926325 b0ea8000 Options.recycle_log_file_num: 0
2017/03/20-20:36:05.926326 b0ea8000 Options.allow_os_buffer: 1
2017/03/20-20:36:05.926327 b0ea8000 Options.allow_mmap_reads: 0
2017/03/20-20:36:05.926328 b0ea8000 Options.allow_fallocate: 1
2017/03/20-20:36:05.926329 b0ea8000 Options.allow_mmap_writes: 0
2017/03/20-20:36:05.926330 b0ea8000 Options.create_missing_column_families: 0
2017/03/20-20:36:05.926331 b0ea8000 Options.db_log_dir:
2017/03/20-20:36:05.926332 b0ea8000 Options.wal_dir: some_db.rocks
2017/03/20-20:36:05.926333 b0ea8000 Options.table_cache_numshardbits: 6
2017/03/20-20:36:05.926334 b0ea8000 Options.delete_obsolete_files_period_micros: 21600000000
2017/03/20-20:36:05.926335 b0ea8000 Options.max_subcompactions: 1
2017/03/20-20:36:05.926336 b0ea8000 Options.max_background_flushes: 1
2017/03/20-20:36:05.926337 b0ea8000 Options.WAL_ttl_seconds: 0
2017/03/20-20:36:05.926337 b0ea8000 Options.WAL_size_limit_MB: 0
2017/03/20-20:36:05.926338 b0ea8000 Options.manifest_preallocation_size: 4194304
2017/03/20-20:36:05.926339 b0ea8000 Options.allow_os_buffer: 1
2017/03/20-20:36:05.926340 b0ea8000 Options.allow_mmap_reads: 0
2017/03/20-20:36:05.926341 b0ea8000 Options.allow_mmap_writes: 0
2017/03/20-20:36:05.926342 b0ea8000 Options.is_fd_close_on_exec: 1
2017/03/20-20:36:05.926343 b0ea8000 Options.stats_dump_period_sec: 600
2017/03/20-20:36:05.926344 b0ea8000 Options.advise_random_on_open: 1
2017/03/20-20:36:05.926345 b0ea8000 Options.db_write_buffer_size: 0
2017/03/20-20:36:05.926346 b0ea8000 Options.access_hint_on_compaction_start: 1
2017/03/20-20:36:05.926347 b0ea8000 Options.new_table_reader_for_compaction_inputs: 0
2017/03/20-20:36:05.926348 b0ea8000 Options.compaction_readahead_size: 0
2017/03/20-20:36:05.926349 b0ea8000 Options.random_access_max_buffer_size: 1048576
2017/03/20-20:36:05.926350 b0ea8000 Options.writable_file_max_buffer_size: 1048576
2017/03/20-20:36:05.926351 b0ea8000 Options.use_adaptive_mutex: 0
2017/03/20-20:36:05.926359 b0ea8000 Options.rate_limiter: 0x0
2017/03/20-20:36:05.926361 b0ea8000 Options.sst_file_manager.rate_bytes_per_sec: 0
2017/03/20-20:36:05.926362 b0ea8000 Options.bytes_per_sync: 0
2017/03/20-20:36:05.926363 b0ea8000 Options.wal_bytes_per_sync: 0
2017/03/20-20:36:05.926364 b0ea8000 Options.wal_recovery_mode: 2
2017/03/20-20:36:05.926365 b0ea8000 Options.enable_thread_tracking: 0
2017/03/20-20:36:05.926366 b0ea8000 Options.delayed_write_rate : 2097152
2017/03/20-20:36:05.926367 b0ea8000 Options.allow_concurrent_memtable_write: 0
2017/03/20-20:36:05.926367 b0ea8000 Options.enable_write_thread_adaptive_yield: 0
2017/03/20-20:36:05.926368 b0ea8000 Options.write_thread_max_yield_usec: 100
2017/03/20-20:36:05.926369 b0ea8000 Options.write_thread_slow_yield_usec: 3
2017/03/20-20:36:05.926370 b0ea8000 Options.row_cache: None
2017/03/20-20:36:05.926371 b0ea8000 Options.wal_filter: None
2017/03/20-20:36:05.926372 b0ea8000 Options.avoid_flush_during_recovery: 0
2017/03/20-20:36:05.926373 b0ea8000 Options.base_background_compactions: 1
2017/03/20-20:36:05.926374 b0ea8000 Options.max_background_compactions: 1
2017/03/20-20:36:05.926376 b0ea8000 Compression algorithms supported:
2017/03/20-20:36:05.926377 b0ea8000 Snappy supported: 1
2017/03/20-20:36:05.926377 b0ea8000 Zlib supported: 0
2017/03/20-20:36:05.926378 b0ea8000 Bzip supported: 0
2017/03/20-20:36:05.926379 b0ea8000 LZ4 supported: 0
2017/03/20-20:36:05.926380 b0ea8000 Fast CRC32 supported: 0
2017/03/20-20:36:05.926466 b0ea8000 Creating manifest 1
2017/03/20-20:36:05.933995 b0ea8000 Recovering from manifest file: MANIFEST-000001
2017/03/20-20:36:05.934045 b0ea8000 --------------- Options for column family [default]:
2017/03/20-20:36:05.934051 b0ea8000 Options.comparator: rocksdb.InternalKeyComparator:leveldb.BytewiseComparator
2017/03/20-20:36:05.934055 b0ea8000 Options.merge_operator: None
2017/03/20-20:36:05.934057 b0ea8000 Options.compaction_filter: None
2017/03/20-20:36:05.934058 b0ea8000 Options.compaction_filter_factory: None
2017/03/20-20:36:05.934060 b0ea8000 Options.memtable_factory: SkipListFactory
2017/03/20-20:36:05.934062 b0ea8000 Options.table_factory: BlockBasedTable
2017/03/20-20:36:05.934072 b0ea8000 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fd4ff4a1de0)
cache_index_and_filter_blocks: 0
pin_l0_filter_and_index_blocks_in_cache: 0
index_type: 0
hash_index_allow_collision: 1
checksum: 1
no_block_cache: 0
block_cache: 0x7fd4ff4a7e68
block_cache_size: 8388608
block_cache_compressed: 0x0
block_size: 4096
block_size_deviation: 10
block_restart_interval: 16
index_block_restart_interval: 1
filter_policy: nullptr
whole_key_filtering: 1
skip_table_builder_flush: 0
format_version: 2
2017/03/20-20:36:05.934077 b0ea8000 Options.write_buffer_size: 67108864
2017/03/20-20:36:05.934079 b0ea8000 Options.max_write_buffer_number: 2
2017/03/20-20:36:05.934080 b0ea8000 Options.compression: Snappy
2017/03/20-20:36:05.934082 b0ea8000 Options.bottommost_compression: Disabled
2017/03/20-20:36:05.934083 b0ea8000 Options.prefix_extractor: nullptr
2017/03/20-20:36:05.934085 b0ea8000 Options.num_levels: 7
2017/03/20-20:36:05.934086 b0ea8000 Options.min_write_buffer_number_to_merge: 1
2017/03/20-20:36:05.934087 b0ea8000 Options.max_write_buffer_number_to_maintain: 0
2017/03/20-20:36:05.934089 b0ea8000 Options.compression_opts.window_bits: -14
2017/03/20-20:36:05.934090 b0ea8000 Options.compression_opts.level: -1
2017/03/20-20:36:05.934092 b0ea8000 Options.compression_opts.strategy: 0
2017/03/20-20:36:05.934093 b0ea8000 Options.compression_opts.max_dict_bytes: 0
2017/03/20-20:36:05.934111 b0ea8000 Options.level0_file_num_compaction_trigger: 4
2017/03/20-20:36:05.934113 b0ea8000 Options.level0_slowdown_writes_trigger: 20
2017/03/20-20:36:05.934114 b0ea8000 Options.level0_stop_writes_trigger: 24
2017/03/20-20:36:05.934116 b0ea8000 Options.target_file_size_base: 67108864
2017/03/20-20:36:05.934117 b0ea8000 Options.target_file_size_multiplier: 1
2017/03/20-20:36:05.934119 b0ea8000 Options.max_bytes_for_level_base: 268435456
2017/03/20-20:36:05.934120 b0ea8000 Options.level_compaction_dynamic_level_bytes: 0
2017/03/20-20:36:05.934121 b0ea8000 Options.max_bytes_for_level_multiplier: 10
2017/03/20-20:36:05.934123 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[0]: 1
2017/03/20-20:36:05.934124 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[1]: 1
2017/03/20-20:36:05.934126 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[2]: 1
2017/03/20-20:36:05.934127 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[3]: 1
2017/03/20-20:36:05.934129 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[4]: 1
2017/03/20-20:36:05.934130 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[5]: 1
2017/03/20-20:36:05.934131 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[6]: 1
2017/03/20-20:36:05.934133 b0ea8000 Options.max_sequential_skip_in_iterations: 8
2017/03/20-20:36:05.934134 b0ea8000 Options.max_compaction_bytes: 1677721600
2017/03/20-20:36:05.934136 b0ea8000 Options.arena_block_size: 8388608
2017/03/20-20:36:05.934137 b0ea8000 Options.soft_pending_compaction_bytes_limit: 68719476736
2017/03/20-20:36:05.934139 b0ea8000 Options.hard_pending_compaction_bytes_limit: 274877906944
2017/03/20-20:36:05.934140 b0ea8000 Options.rate_limit_delay_max_milliseconds: 1000
2017/03/20-20:36:05.934142 b0ea8000 Options.disable_auto_compactions: 0
2017/03/20-20:36:05.934143 b0ea8000 Options.verify_checksums_in_compaction: 1
2017/03/20-20:36:05.934144 b0ea8000 Options.compaction_style: 0
2017/03/20-20:36:05.934146 b0ea8000 Options.compaction_pri: 0
2017/03/20-20:36:05.934147 b0ea8000 Options.compaction_options_universal.size_ratio: 1
2017/03/20-20:36:05.934149 b0ea8000 Options.compaction_options_universal.min_merge_width: 2
2017/03/20-20:36:05.934150 b0ea8000 Options.compaction_options_universal.max_merge_width: 4294967295
2017/03/20-20:36:05.934151 b0ea8000 Options.compaction_options_universal.max_size_amplification_percent: 200
2017/03/20-20:36:05.934153 b0ea8000 Options.compaction_options_universal.compression_size_percent: -1
2017/03/20-20:36:05.934154 b0ea8000 Options.compaction_options_fifo.max_table_files_size: 1073741824
2017/03/20-20:36:05.934156 b0ea8000 Options.table_properties_collectors:
2017/03/20-20:36:05.934157 b0ea8000 Options.inplace_update_support: 0
2017/03/20-20:36:05.934159 b0ea8000 Options.inplace_update_num_locks: 10000
2017/03/20-20:36:05.934160 b0ea8000 Options.min_partial_merge_operands: 2
2017/03/20-20:36:05.934162 b0ea8000 Options.memtable_prefix_bloom_size_ratio: 0.000000
2017/03/20-20:36:05.934176 b0ea8000 Options.memtable_huge_page_size: 0
2017/03/20-20:36:05.934177 b0ea8000 Options.bloom_locality: 0
2017/03/20-20:36:05.934179 b0ea8000 Options.max_successive_merges: 0
2017/03/20-20:36:05.934180 b0ea8000 Options.optimize_filters_for_hits: 0
2017/03/20-20:36:05.934181 b0ea8000 Options.paranoid_file_checks: 0
2017/03/20-20:36:05.934183 b0ea8000 Options.force_consistency_checks: 0
2017/03/20-20:36:05.934184 b0ea8000 Options.report_bg_io_stats: 0
2017/03/20-20:36:05.934709 b0ea8000 Recovered from manifest file:some_db.rocks/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0
2017/03/20-20:36:05.934731 b0ea8000 Column family [default] (ID 0), log number is 0
2017/03/20-20:36:05.936975 b0ea8000 DB pointer 0x7fd4ff8c9000
2017/03/20-20:36:07.746866 b0ea8000 Creating manifest 6
2017/03/20-20:36:07.749553 b0ea8000 --------------- Options for column family [people]:
2017/03/20-20:36:07.749561 b0ea8000 Options.comparator: rocksdb.InternalKeyComparator:leveldb.BytewiseComparator
2017/03/20-20:36:07.749563 b0ea8000 Options.merge_operator: None
2017/03/20-20:36:07.749565 b0ea8000 Options.compaction_filter: None
2017/03/20-20:36:07.749566 b0ea8000 Options.compaction_filter_factory: None
2017/03/20-20:36:07.749568 b0ea8000 Options.memtable_factory: SkipListFactory
2017/03/20-20:36:07.749570 b0ea8000 Options.table_factory: BlockBasedTable
2017/03/20-20:36:07.749581 b0ea8000 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fd501a00550)
cache_index_and_filter_blocks: 0
pin_l0_filter_and_index_blocks_in_cache: 0
index_type: 0
hash_index_allow_collision: 1
checksum: 1
no_block_cache: 0
block_cache: 0x7fd501a00598
block_cache_size: 8388608
block_cache_compressed: 0x0
block_size: 4096
block_size_deviation: 10
block_restart_interval: 16
index_block_restart_interval: 1
filter_policy: nullptr
whole_key_filtering: 1
skip_table_builder_flush: 0
format_version: 2
2017/03/20-20:36:07.749587 b0ea8000 Options.write_buffer_size: 67108864
2017/03/20-20:36:07.749589 b0ea8000 Options.max_write_buffer_number: 2
2017/03/20-20:36:07.749591 b0ea8000 Options.compression: Snappy
2017/03/20-20:36:07.749593 b0ea8000 Options.bottommost_compression: Disabled
2017/03/20-20:36:07.749594 b0ea8000 Options.prefix_extractor: nullptr
2017/03/20-20:36:07.749596 b0ea8000 Options.num_levels: 7
2017/03/20-20:36:07.749597 b0ea8000 Options.min_write_buffer_number_to_merge: 1
2017/03/20-20:36:07.749599 b0ea8000 Options.max_write_buffer_number_to_maintain: 0
2017/03/20-20:36:07.749600 b0ea8000 Options.compression_opts.window_bits: -14
2017/03/20-20:36:07.749602 b0ea8000 Options.compression_opts.level: -1
2017/03/20-20:36:07.749603 b0ea8000 Options.compression_opts.strategy: 0
2017/03/20-20:36:07.749605 b0ea8000 Options.compression_opts.max_dict_bytes: 0
2017/03/20-20:36:07.749606 b0ea8000 Options.level0_file_num_compaction_trigger: 4
2017/03/20-20:36:07.749608 b0ea8000 Options.level0_slowdown_writes_trigger: 20
2017/03/20-20:36:07.749609 b0ea8000 Options.level0_stop_writes_trigger: 24
2017/03/20-20:36:07.749611 b0ea8000 Options.target_file_size_base: 67108864
2017/03/20-20:36:07.749613 b0ea8000 Options.target_file_size_multiplier: 1
2017/03/20-20:36:07.749614 b0ea8000 Options.max_bytes_for_level_base: 268435456
2017/03/20-20:36:07.749616 b0ea8000 Options.level_compaction_dynamic_level_bytes: 0
2017/03/20-20:36:07.749617 b0ea8000 Options.max_bytes_for_level_multiplier: 10
2017/03/20-20:36:07.749618 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[0]: 1
2017/03/20-20:36:07.749620 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[1]: 1
2017/03/20-20:36:07.749622 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[2]: 1
2017/03/20-20:36:07.749623 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[3]: 1
2017/03/20-20:36:07.749625 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[4]: 1
2017/03/20-20:36:07.749626 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[5]: 1
2017/03/20-20:36:07.749628 b0ea8000 Options.max_bytes_for_level_multiplier_addtl[6]: 1
2017/03/20-20:36:07.749629 b0ea8000 Options.max_sequential_skip_in_iterations: 8
2017/03/20-20:36:07.749631 b0ea8000 Options.max_compaction_bytes: 1677721600
2017/03/20-20:36:07.749632 b0ea8000 Options.arena_block_size: 8388608
2017/03/20-20:36:07.749634 b0ea8000 Options.soft_pending_compaction_bytes_limit: 68719476736
2017/03/20-20:36:07.749636 b0ea8000 Options.hard_pending_compaction_bytes_limit: 274877906944
2017/03/20-20:36:07.749638 b0ea8000 Options.rate_limit_delay_max_milliseconds: 1000
2017/03/20-20:36:07.749640 b0ea8000 Options.disable_auto_compactions: 0
2017/03/20-20:36:07.749654 b0ea8000 Options.verify_checksums_in_compaction: 1
2017/03/20-20:36:07.749657 b0ea8000 Options.compaction_style: 0
2017/03/20-20:36:07.749659 b0ea8000 Options.compaction_pri: 0
2017/03/20-20:36:07.749661 b0ea8000 Options.compaction_options_universal.size_ratio: 1
2017/03/20-20:36:07.749663 b0ea8000 Options.compaction_options_universal.min_merge_width: 2
2017/03/20-20:36:07.749665 b0ea8000 Options.compaction_options_universal.max_merge_width: 4294967295
2017/03/20-20:36:07.749667 b0ea8000 Options.compaction_options_universal.max_size_amplification_percent: 200
2017/03/20-20:36:07.749669 b0ea8000 Options.compaction_options_universal.compression_size_percent: -1
2017/03/20-20:36:07.749671 b0ea8000 Options.compaction_options_fifo.max_table_files_size: 1073741824
2017/03/20-20:36:07.749673 b0ea8000 Options.table_properties_collectors:
2017/03/20-20:36:07.749675 b0ea8000 Options.inplace_update_support: 0
2017/03/20-20:36:07.749676 b0ea8000 Options.inplace_update_num_locks: 10000
2017/03/20-20:36:07.749678 b0ea8000 Options.min_partial_merge_operands: 2
2017/03/20-20:36:07.749679 b0ea8000 Options.memtable_prefix_bloom_size_ratio: 0.000000
2017/03/20-20:36:07.749681 b0ea8000 Options.memtable_huge_page_size: 0
2017/03/20-20:36:07.749683 b0ea8000 Options.bloom_locality: 0
2017/03/20-20:36:07.749685 b0ea8000 Options.max_successive_merges: 0
2017/03/20-20:36:07.749686 b0ea8000 Options.optimize_filters_for_hits: 0
2017/03/20-20:36:07.749688 b0ea8000 Options.paranoid_file_checks: 0
2017/03/20-20:36:07.749690 b0ea8000 Options.force_consistency_checks: 0
2017/03/20-20:36:07.749692 b0ea8000 Options.report_bg_io_stats: 0
2017/03/20-20:36:07.752664 b0ea8000 Created column family [people] (ID 1)

Binary file not shown.

Binary file not shown.

View File

@ -1,135 +0,0 @@
# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#
[Version]
rocksdb_version=4.13.0
options_file_version=1.1
[DBOptions]
avoid_flush_during_recovery=false
dump_malloc_stats=false
info_log_level=INFO_LEVEL
access_hint_on_compaction_start=NORMAL
write_thread_max_yield_usec=100
write_thread_slow_yield_usec=3
enable_write_thread_adaptive_yield=false
allow_os_buffer=true
fail_if_options_file_error=false
max_log_file_size=0
stats_dump_period_sec=600
max_manifest_file_size=18446744073709551615
bytes_per_sync=0
delayed_write_rate=2097152
WAL_ttl_seconds=0
allow_concurrent_memtable_write=false
paranoid_checks=true
writable_file_max_buffer_size=1048576
WAL_size_limit_MB=0
max_subcompactions=1
wal_dir=some_db.rocks
wal_bytes_per_sync=0
max_total_wal_size=0
db_write_buffer_size=0
keep_log_file_num=1000
table_cache_numshardbits=6
max_file_opening_threads=16
random_access_max_buffer_size=1048576
use_fsync=false
max_open_files=-1
skip_stats_update_on_db_open=false
max_background_compactions=1
error_if_exists=false
manifest_preallocation_size=4194304
max_background_flushes=1
is_fd_close_on_exec=true
advise_random_on_open=true
create_missing_column_families=false
delete_obsolete_files_period_micros=21600000000
create_if_missing=true
disable_data_sync=false
log_file_time_to_roll=0
compaction_readahead_size=0
use_adaptive_mutex=false
recycle_log_file_num=0
skip_log_error_on_recovery=false
allow_mmap_reads=false
disableDataSync=false
enable_thread_tracking=false
allow_2pc=false
allow_fallocate=true
wal_recovery_mode=kPointInTimeRecovery
db_log_dir=
new_table_reader_for_compaction_inputs=false
base_background_compactions=1
allow_mmap_writes=false
[CFOptions "default"]
compaction_style=kCompactionStyleLevel
merge_operator=nullptr
compaction_filter=nullptr
memtable_factory=SkipListFactory
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
max_sequential_skip_in_iterations=8
comparator=leveldb.BytewiseComparator
max_bytes_for_level_base=268435456
soft_pending_compaction_bytes_limit=68719476736
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
min_write_buffer_number_to_merge=1
target_file_size_multiplier=1
compression_per_level=
table_factory=BlockBasedTable
num_levels=7
level0_stop_writes_trigger=24
compression=kSnappyCompression
level0_file_num_compaction_trigger=4
inplace_update_num_locks=10000
level_compaction_dynamic_level_bytes=false
level0_slowdown_writes_trigger=20
verify_checksums_in_compaction=true
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
max_write_buffer_number=2
force_consistency_checks=false
min_partial_merge_operands=2
memtable_prefix_bloom_size_ratio=0.000000
paranoid_file_checks=false
hard_pending_compaction_bytes_limit=274877906944
max_write_buffer_number_to_maintain=0
purge_redundant_kvs_while_flush=true
target_file_size_base=67108864
optimize_filters_for_hits=false
compaction_filter_factory=nullptr
max_bytes_for_level_multiplier=10
bloom_locality=0
write_buffer_size=67108864
disable_auto_compactions=false
max_compaction_bytes=1677721600
inplace_update_support=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "default"]
verify_compression=false
format_version=2
whole_key_filtering=true
index_block_restart_interval=1
block_size_deviation=10
block_size=4096
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
skip_table_builder_flush=false
filter_policy=nullptr
no_block_cache=false
checksum=kCRC32c
read_amp_bytes_per_bit=8589934592
cache_index_and_filter_blocks=false
index_type=kBinarySearch
hash_index_allow_collision=true
cache_index_and_filter_blocks_with_high_priority=false
flush_block_policy_factory=FlushBlockBySizePolicyFactory

View File

@ -1,202 +0,0 @@
# This is a RocksDB option file.
#
# For detailed file format spec, please refer to the example file
# in examples/rocksdb_option_file_example.ini
#
[Version]
rocksdb_version=4.13.0
options_file_version=1.1
[DBOptions]
avoid_flush_during_recovery=false
dump_malloc_stats=false
info_log_level=INFO_LEVEL
access_hint_on_compaction_start=NORMAL
write_thread_max_yield_usec=100
write_thread_slow_yield_usec=3
enable_write_thread_adaptive_yield=false
allow_os_buffer=true
fail_if_options_file_error=false
max_log_file_size=0
stats_dump_period_sec=600
max_manifest_file_size=18446744073709551615
bytes_per_sync=0
delayed_write_rate=2097152
WAL_ttl_seconds=0
allow_concurrent_memtable_write=false
paranoid_checks=true
writable_file_max_buffer_size=1048576
WAL_size_limit_MB=0
max_subcompactions=1
wal_dir=some_db.rocks
wal_bytes_per_sync=0
max_total_wal_size=0
db_write_buffer_size=0
keep_log_file_num=1000
table_cache_numshardbits=6
max_file_opening_threads=16
random_access_max_buffer_size=1048576
use_fsync=false
max_open_files=-1
skip_stats_update_on_db_open=false
max_background_compactions=1
error_if_exists=false
manifest_preallocation_size=4194304
max_background_flushes=1
is_fd_close_on_exec=true
advise_random_on_open=true
create_missing_column_families=false
delete_obsolete_files_period_micros=21600000000
create_if_missing=true
disable_data_sync=false
log_file_time_to_roll=0
compaction_readahead_size=0
use_adaptive_mutex=false
recycle_log_file_num=0
skip_log_error_on_recovery=false
allow_mmap_reads=false
disableDataSync=false
enable_thread_tracking=false
allow_2pc=false
allow_fallocate=true
wal_recovery_mode=kPointInTimeRecovery
db_log_dir=
new_table_reader_for_compaction_inputs=false
base_background_compactions=1
allow_mmap_writes=false
[CFOptions "default"]
compaction_style=kCompactionStyleLevel
merge_operator=nullptr
compaction_filter=nullptr
memtable_factory=SkipListFactory
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
max_sequential_skip_in_iterations=8
comparator=leveldb.BytewiseComparator
max_bytes_for_level_base=268435456
soft_pending_compaction_bytes_limit=68719476736
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
min_write_buffer_number_to_merge=1
target_file_size_multiplier=1
compression_per_level=
table_factory=BlockBasedTable
num_levels=7
level0_stop_writes_trigger=24
compression=kSnappyCompression
level0_file_num_compaction_trigger=4
inplace_update_num_locks=10000
level_compaction_dynamic_level_bytes=false
level0_slowdown_writes_trigger=20
verify_checksums_in_compaction=true
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
max_write_buffer_number=2
force_consistency_checks=false
min_partial_merge_operands=2
memtable_prefix_bloom_size_ratio=0.000000
paranoid_file_checks=false
hard_pending_compaction_bytes_limit=274877906944
max_write_buffer_number_to_maintain=0
purge_redundant_kvs_while_flush=true
target_file_size_base=67108864
optimize_filters_for_hits=false
compaction_filter_factory=nullptr
max_bytes_for_level_multiplier=10
bloom_locality=0
write_buffer_size=67108864
disable_auto_compactions=false
max_compaction_bytes=1677721600
inplace_update_support=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "default"]
verify_compression=false
format_version=2
whole_key_filtering=true
index_block_restart_interval=1
block_size_deviation=10
block_size=4096
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
skip_table_builder_flush=false
filter_policy=nullptr
no_block_cache=false
checksum=kCRC32c
read_amp_bytes_per_bit=8589934592
cache_index_and_filter_blocks=false
index_type=kBinarySearch
hash_index_allow_collision=true
cache_index_and_filter_blocks_with_high_priority=false
flush_block_policy_factory=FlushBlockBySizePolicyFactory
[CFOptions "people"]
compaction_style=kCompactionStyleLevel
merge_operator=nullptr
compaction_filter=nullptr
memtable_factory=SkipListFactory
prefix_extractor=nullptr
bottommost_compression=kDisableCompressionOption
max_sequential_skip_in_iterations=8
comparator=leveldb.BytewiseComparator
max_bytes_for_level_base=268435456
soft_pending_compaction_bytes_limit=68719476736
memtable_huge_page_size=0
max_successive_merges=0
arena_block_size=8388608
min_write_buffer_number_to_merge=1
target_file_size_multiplier=1
compression_per_level=
table_factory=BlockBasedTable
num_levels=7
level0_stop_writes_trigger=24
compression=kSnappyCompression
level0_file_num_compaction_trigger=4
inplace_update_num_locks=10000
level_compaction_dynamic_level_bytes=false
level0_slowdown_writes_trigger=20
verify_checksums_in_compaction=true
max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
max_write_buffer_number=2
force_consistency_checks=false
min_partial_merge_operands=2
memtable_prefix_bloom_size_ratio=0.000000
paranoid_file_checks=false
hard_pending_compaction_bytes_limit=274877906944
max_write_buffer_number_to_maintain=0
purge_redundant_kvs_while_flush=true
target_file_size_base=67108864
optimize_filters_for_hits=false
compaction_filter_factory=nullptr
max_bytes_for_level_multiplier=10
bloom_locality=0
write_buffer_size=67108864
disable_auto_compactions=false
max_compaction_bytes=1677721600
inplace_update_support=false
report_bg_io_stats=false
[TableOptions/BlockBasedTable "people"]
verify_compression=false
format_version=2
whole_key_filtering=true
index_block_restart_interval=1
block_size_deviation=10
block_size=4096
pin_l0_filter_and_index_blocks_in_cache=false
block_restart_interval=16
skip_table_builder_flush=false
filter_policy=nullptr
no_block_cache=false
checksum=kCRC32c
read_amp_bytes_per_bit=8589934592
cache_index_and_filter_blocks=false
index_type=kBinarySearch
hash_index_allow_collision=true
cache_index_and_filter_blocks_with_high_priority=false
flush_block_policy_factory=FlushBlockBySizePolicyFactory

View File

@ -1 +0,0 @@
ryan@amplitude.local.94644

View File

@ -3,8 +3,17 @@ defmodule RoxTest do
doctest Rox
setup do
path = Path.join(__DIR__, "test.rocksdb")
{:ok, db} = Rox.open(path, create_if_missing: true)
path =
Path.join(__DIR__, "test.rocksdb")
cf_path =
Path.join(__DIR__, "cf_test.rocksdb")
{:ok, db} =
Rox.open(path, create_if_missing: true)
{:ok, db} =
Rox.open(path, create_if_missing: true)
on_exit fn ->
Rox.close(db)

17
test/utils_test.exs Normal file
View File

@ -0,0 +1,17 @@
defmodule Rox.UtilsTest do
use ExUnit.Case, async: true
import Rox.Utils
test "encode / decode cycle" do
input =
%{name: "Bob"}
output =
input
|> encode
|> decode
assert input == output
end
end