zecfoundation
/
metrics
mirror of https://github.com/ZcashFoundation/metrics.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix streaming bug + no_std attempt with util as the testbench

This commit is contained in:
Toby Lawrence 2020-10-24 09:08:11 -04:00
parent 91be4d0aad
commit b8bee9b19e
74 changed files with 111 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
metrics-util/Cargo.toml
View File

@ -32,23 +32,27 @@ harness = false
[dependencies]
metrics = { version = "0.13.0-alpha.1", path = "../metrics", features = ["std"] }
crossbeam-epoch = "0.9"
crossbeam-utils = "0.8"
serde = "1.0"
arc-swap = "0.4"
atomic-shim = "0.1"
parking_lot = "0.11"
crossbeam-epoch = { version = "0.9", optional = true }
crossbeam-utils = { version = "0.8", default-features = false }
arc-swap = { version = "0.4", optional = true }
atomic-shim = { version = "0.1", optional = true }
aho-corasick = { version = "0.7", optional = true }
dashmap = "3"
indexmap = "1.6"
dashmap = { version = "3", optional = true }
indexmap = { version = "1.6", optional = true }}
[dev-dependencies]
bolero = "0.5"
criterion = "0.3"
lazy_static = "1.3"
rand = { version = "0.7", features = ["small_rng"] }
rand_distr = "0.3"
[features]
default = ["std", "layer-filter"]
std = []
default = ["std"]
std = ["arc-swap", "atomic-shim", "crossbeam-epoch", "dashmap", "indexmap"]
layer-filter = ["aho-corasick"]
[[test]]
name = "streaming"
path = "tests/streaming/fuzz_target.rs"
harness = false

2
metrics-util/src/debugging.rs
View File

@ -1,5 +1,5 @@
use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use core::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex};
use crate::{handle::Handle, registry::Registry};

15
metrics-util/src/lib.rs
View File

@ -1,24 +1,33 @@
//! Helper types and functions used within the metrics ecosystem.
#![deny(missing_docs)]
#![cfg_attr(not(feature = "std"), no_std)]
#[cfg(feature = "std")]
mod bucket;
#[cfg(feature = "std")]
pub use bucket::AtomicBucket;
#[cfg(feature = "std")]
mod debugging;
#[cfg(feature = "std")]
pub use debugging::{DebugValue, DebuggingRecorder, MetricKind, Snapshotter};
#[cfg(feature = "std")]
mod handle;
#[cfg(feature = "std")]
pub use handle::Handle;
#[cfg(feature = "std")]
mod streaming;
#[cfg(feature = "std")]
pub use streaming::StreamingIntegers;
mod quantile;
pub use quantile::{parse_quantiles, Quantile};
mod tree;
pub use tree::{Integer, MetricsTree};
#[cfg(feature = "std")]
mod registry;
#[cfg(feature = "std")]
pub use registry::Registry;
mod key;

2
metrics-util/src/registry.rs
View File

@ -1,6 +1,6 @@
use dashmap::DashMap;
use std::collections::HashMap;
use std::hash::Hash;
use core::hash::Hash;
/// A high-performance metric registry.
///

39
metrics-util/src/streaming.rs
View File

@ -1,4 +1,4 @@
use std::slice;
use core::slice;
/// A compressed set of integers.
///
@ -50,7 +50,7 @@ use std::slice;
pub struct StreamingIntegers {
inner: Vec<u8>,
len: usize,
last: Option<i64>,
last: Option<i128>,
}
impl StreamingIntegers {
@ -99,7 +99,7 @@ impl StreamingIntegers {
// a delta value.
let mut src_idx = 0;
if self.last.is_none() {
let first = src[src_idx] as i64;
let first = src[src_idx] as i128;
self.last = Some(first);
let zigzag = zigzag_encode(first);
@ -112,7 +112,8 @@ impl StreamingIntegers {
let mut last = self.last.unwrap();
while src_idx < src_len {
let value = src[src_idx] as i64;
let value = src[src_idx] as i128;
// attempted to subtract with overflow
let diff = value - last;
let zigzag = zigzag_encode(diff);
buf_idx = vbyte_encode(zigzag, &mut buf, buf_idx);
@ -194,17 +195,17 @@ impl StreamingIntegers {
}
#[inline]
fn zigzag_encode(input: i64) -> u64 {
((input << 1) ^ (input >> 63)) as u64
fn zigzag_encode(input: i128) -> u128 {
((input << 1) ^ (input >> 127)) as u128
}
#[inline]
fn zigzag_decode(input: u64) -> i64 {
((input >> 1) as i64) ^ (-((input & 1) as i64))
fn zigzag_decode(input: u128) -> i128 {
((input >> 1) as i128) ^ (-((input & 1) as i128))
}
#[inline]
fn vbyte_encode(mut input: u64, buf: &mut [u8], mut buf_idx: usize) -> usize {
fn vbyte_encode(mut input: u128, buf: &mut [u8], mut buf_idx: usize) -> usize {
while input >= 128 {
buf[buf_idx] = 0x80 as u8 | (input as u8 & 0x7F);
buf_idx += 1;
@ -215,11 +216,11 @@ fn vbyte_encode(mut input: u64, buf: &mut [u8], mut buf_idx: usize) -> usize {
}
#[inline]
fn vbyte_decode(buf: &[u8], mut buf_idx: usize) -> (u64, usize) {
fn vbyte_decode(buf: &[u8], mut buf_idx: usize) -> (u128, usize) {
let mut tmp = 0;
let mut factor = 0;
loop {
tmp |= u64::from(buf[buf_idx] & 0x7F) << (7 * factor);
tmp |= u128::from(buf[buf_idx] & 0x7F) << (7 * factor);
if buf[buf_idx] & 0x80 != 0x80 {
return (tmp, buf_idx + 1);
}
@ -240,6 +241,22 @@ mod tests {
assert_eq!(decompressed.len(), 0);
}
#[test]
fn test_streaming_integers_edge_cases() {
let mut si = StreamingIntegers::new();
let decompressed = si.decompress();
assert_eq!(decompressed.len(), 0);
let values = vec![
140754668284938,
9223372079804448768,
];
si.compress(&values);
let decompressed = si.decompress();
assert_eq!(decompressed, values);
}
#[test]
fn test_streaming_integers_single_block() {
let mut si = StreamingIntegers::new();

122
metrics-util/src/tree.rs
View File

@ -1,122 +0,0 @@
use serde::ser::{Serialize, Serializer};
use std::collections::HashMap;
/// An integer metric value.
pub enum Integer {
/// A signed value.
Signed(i64),
/// An unsigned value.
Unsigned(u64),
}
impl From<i64> for Integer {
fn from(i: i64) -> Integer {
Integer::Signed(i)
}
}
impl From<u64> for Integer {
fn from(i: u64) -> Integer {
Integer::Unsigned(i)
}
}
enum TreeEntry {
Value(Integer),
Nested(MetricsTree),
}
impl Serialize for TreeEntry {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match self {
TreeEntry::Value(value) => match value {
Integer::Signed(i) => serializer.serialize_i64(*i),
Integer::Unsigned(i) => serializer.serialize_u64(*i),
},
TreeEntry::Nested(tree) => tree.serialize(serializer),
}
}
}
/// A tree-structured metrics container.
///
/// Used for building a tree structure out of scoped metrics, where each level in the tree
/// represents a nested scope.
#[derive(Default)]
pub struct MetricsTree {
contents: HashMap<String, TreeEntry>,
}
impl MetricsTree {
/// Inserts a single value into the tree.
pub fn insert_value<V: Into<Integer>>(
&mut self,
mut levels: Vec<String>,
key: String,
value: V,
) {
match levels.len() {
0 => {
self.contents.insert(key, TreeEntry::Value(value.into()));
}
_ => {
let name = levels.remove(0);
let inner = self
.contents
.entry(name)
.or_insert_with(|| TreeEntry::Nested(MetricsTree::default()));
if let TreeEntry::Nested(tree) = inner {
tree.insert_value(levels, key, value);
}
}
}
}
/// Inserts multiple values into the tree.
pub fn insert_values<V: Into<Integer>>(
&mut self,
mut levels: Vec<String>,
values: Vec<(String, V)>,
) {
match levels.len() {
0 => {
for v in values.into_iter() {
self.contents.insert(v.0, TreeEntry::Value(v.1.into()));
}
}
_ => {
let name = levels.remove(0);
let inner = self
.contents
.entry(name)
.or_insert_with(|| TreeEntry::Nested(MetricsTree::default()));
if let TreeEntry::Nested(tree) = inner {
tree.insert_values(levels, values);
}
}
}
}
/// Clears all entries in the tree.
pub fn clear(&mut self) {
self.contents.clear();
}
}
impl Serialize for MetricsTree {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut sorted = self.contents.iter().collect::<Vec<_>>();
sorted.sort_by_key(|p| p.0);
serializer.collect_map(sorted)
}
}

1
metrics-util/tests/streaming/corpus/0ba3c96ead62cb9b9b3b1caa48a3b4d60f11e8ad Normal file
View File

@ -0,0 +1 @@
糜<EFBFBD>箒筱sssｪ>

1
metrics-util/tests/streaming/corpus/0f736fe9119c0eb5b75dd0affd5e38ac43d6553e Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>s<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ss<EFBFBD>><3E><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

1
metrics-util/tests/streaming/corpus/15ee00db03dc8bc4fe6bf2e61ef0494c327d39bf Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD>鮊<EFBFBD>N

1
metrics-util/tests/streaming/corpus/170a824e7436db8ac8ccc805650aee654bf6b0bf Normal file
View File

@ -0,0 +1 @@
<EFBFBD>徉<EFBFBD><EFBFBD><EFBFBD><EFBFBD>瞁

BIN
metrics-util/tests/streaming/corpus/170b901f1da835afd5e47f924255e7cf9bebb4d7 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/1b9382ea0b0109b40e7c76dae2187c38d1da24c0 Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/1cef3405ced3c5a96639b1246d74b02378050470 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

BIN
metrics-util/tests/streaming/corpus/2529b93f106d5a1b1f1b72a23e0ff27c718d8c87 Normal file
View File

Binary file not shown.

2
metrics-util/tests/streaming/corpus/2e9dc13b9f4ae14debe7c76d30c47eae1b6c438a Normal file
View File

@ -0,0 +1,2 @@
íÿÿÿÿÿÿNíYí«ÍíYí«XXX
Á(y

BIN
metrics-util/tests/streaming/corpus/2f5efbca4af5aad48f624f1507e84c84afae1448 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/30c83ecd8632712dc0f9a385e8edc18e603a8c28 Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/37b62e7428b44c03357de1f196c834a32393b919 Normal file
View File

@ -0,0 +1 @@
ｫ粐粐粐粐粐

BIN
metrics-util/tests/streaming/corpus/388c901b769705d042ab58853eb6268fc8a7fd2f Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/38d347603234288a374b97cadb2fd35faea2d5e2 Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/39d5a277ca7a87b35db17cdc3f72a2edb9b35a45 Normal file
View File

@ -0,0 +1 @@
<EFBFBD>忖

1
metrics-util/tests/streaming/corpus/3b15a514d95c5811c5f185f832bb135764b762ec Normal file
View File

@ -0,0 +1 @@
瞁<EFBFBD><EFBFBD><EFBFBD>薶滫

BIN
metrics-util/tests/streaming/corpus/3c60e2f7e1b86cdfd933f451281c9a3c891b1576 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/3f968be5f30927d64f326f0f98f92df38219f629 Normal file
View File

Binary file not shown.

2
metrics-util/tests/streaming/corpus/3f9b4e008e696aea7189a668a7abed37ee28bfa0 Normal file
View File

@ -0,0 +1,2 @@
<EFBFBD><EFBFBD>N倢戓ﾍXX
ﾁ(y戓(

BIN
metrics-util/tests/streaming/corpus/43ce156ce464bd00347ff65be2308b9f05665122 Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/468bdb2af17b38c2a915e2fc77e09e8f49aaf658 Normal file
View File

@ -0,0 +1 @@
Íí(Y((Y(*

1
metrics-util/tests/streaming/corpus/5213e2566ee6d6de25b6a32cab1b55e045003413 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>s<EFBFBD>s<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

1
metrics-util/tests/streaming/corpus/5873be2cb4a8f99ab332e91f593ec4b30f645f9d Normal file
View File

@ -0,0 +1 @@
<EFBFBD><12><><EFBFBD><EFBFBD>N<EFBFBD><4E>

1
metrics-util/tests/streaming/corpus/592dde5e67f78cbceb3f0692c4690a8bd0a6ad70 Normal file
View File

@ -0,0 +1 @@
ííÍNí˙ĺíA

BIN
metrics-util/tests/streaming/corpus/5a1582fe330481c0820e58b4e73352fc45d30d94 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/5ac9a240bc2763cd09f7bdce55a920c5b12bf286 Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 Normal file
View File

@ -0,0 +1 @@


BIN
metrics-util/tests/streaming/corpus/64376ed93afde6138a1fafe40b9bfce11487bb8d Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/6514dd40c572cf34725f4b92f73fbaa1ce32043d Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

1
metrics-util/tests/streaming/corpus/69c2926955a20d95efcb9598368e03d8051f22b5 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD>鮊<EFBFBD><EFBFBD>鮊

1
metrics-util/tests/streaming/corpus/70153b069d859713491b1e7eb7297fabc166c9c6 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>K

1
metrics-util/tests/streaming/corpus/8168593496b6f261e3ef26359ef50a0273b7a7e5 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><04>

1
metrics-util/tests/streaming/corpus/85e53271e14006f0265921d02d4d736cdc580b0b Normal file
View File

@ -0,0 +1 @@
<EFBFBD>

BIN
metrics-util/tests/streaming/corpus/87bbd7e62282cfa097a5b0deb5815a5189963edf Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/8ac4087f98f25fbaf1ae389b2d69a2cb24ae1b64 Normal file
View File

@ -0,0 +1 @@
澵澵戓ｫ

BIN
metrics-util/tests/streaming/corpus/8e21e08b6b7880a52a07066973365028ab81b5d5 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/9395bdac88ff8b24edc5ba4979636ce42969215e Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/99f0ae11343a02d6ed466985d6a9bc923f80d7d4 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/9a0b73df8c06d962b4455e7a393680a7894a838b Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/9b582a75c61ee9ebed61e18be5fd979bedf88361 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><0E>

BIN
metrics-util/tests/streaming/corpus/9b5c317cdc9c7f2616a31bb3f74a54ccb72995fc Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/9d85a5a384c13c1113d2a99bca076cd944ea8c6f Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/9e423ec6abbd861a3deca4dd57b9eca43a765a5e Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/a083b4df08bc39e125d83c0becd5a4afc769aa54 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>(<28><>(

4
metrics-util/tests/streaming/corpus/a73520580f61edf697cfc40e2a5b4070204572be Normal file
View File

@ -0,0 +1,4 @@
<EFBFBD><EFBFBD>Y#<23><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>&XY<58>XY<58>X<EFBFBD><58><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
X<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
X
<EFBFBD><EFBFBD><EFBFBD><EFBFBD>(<28>y

2
metrics-util/tests/streaming/corpus/a7355be6bb31b3b0839286ec9a4478d8068df2c1 Normal file
View File

@ -0,0 +1,2 @@
ííYí«ÍíYí«XXX
Á(y

BIN
metrics-util/tests/streaming/corpus/a8caf533decb55c76d074f55cade29f4d242c42f Normal file
View File

Binary file not shown.

2
metrics-util/tests/streaming/corpus/aa16c3bfbbe430853f6a5479c7248f50b349fdb2 Normal file
View File

@ -0,0 +1,2 @@
<EFBFBD><EFBFBD>Y<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>&XY<58>X<EFBFBD><58><EFBFBD><EFBFBD><EFBFBD><EFBFBD>X
(<28>y

BIN
metrics-util/tests/streaming/corpus/b7e85c0d539dba9dd4c988056e24c7b56ccb0dd9 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/bf84fa3b8402274293edf6fa5b2c4d94ac2e9623 Normal file
View File

Binary file not shown.

2
metrics-util/tests/streaming/corpus/c21ead1f6b34fbf8f61be224774709c5a58f4100 Normal file
View File

@ -0,0 +1,2 @@
<EFBFBD>
<EFBFBD><EFBFBD>

BIN
metrics-util/tests/streaming/corpus/c259e771b237769cb6bce9a5ab734c576a6da3e1 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/c4ca0a9f3c4f0f518fd11398ddf453bb999f4e94 Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/c8abb0527f9e5f974b68a2dbcc20bf918072f904 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><0E>

BIN
metrics-util/tests/streaming/corpus/cbb3d38709588262548ee8406c44d17e81d0d393 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/d4a25a88e11d3bb556baec588c9999948a7a5508 Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/d5fa40a5ceafd4b822a55930040429bbf98ee198 Normal file
View File

@ -0,0 +1 @@
ｫｫ

1
metrics-util/tests/streaming/corpus/d938a3a869561e8839e5252d284e673f5b9967e7 Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

1
metrics-util/tests/streaming/corpus/dd79c8cfb8beeacd0460429944b4ecbe95a31561 Normal file
View File

@ -0,0 +1 @@
<EFBFBD>

BIN
metrics-util/tests/streaming/corpus/df12318a438a6bbecacdd7b2587d1f536a5350e6 Normal file
View File

Binary file not shown.

BIN
metrics-util/tests/streaming/corpus/e27d4249b690228277e1ba53a57258245ca4357d Normal file
View File

Binary file not shown.

1
metrics-util/tests/streaming/corpus/e36825fabe22e5325c528a2cf525b8a6b1c525f6 Normal file
View File

@ -0,0 +1 @@
ｫ粐z粐粐

1
metrics-util/tests/streaming/corpus/e85d378c9c73e8b52a9b92a4074b73d3b3c09388 Normal file
View File

@ -0,0 +1 @@
迯<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

1
metrics-util/tests/streaming/corpus/e9df434de3ac69a85e8a74c9824f8165a461197f Normal file
View File

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD>鮊<EFBFBD>鮊

3
metrics-util/tests/streaming/corpus/ec87bcc29280f752fab2355d5d353aa418bb5a41 Normal file
View File

@ -0,0 +1,3 @@
ííXX
ÁXXX
Á(yäää)y

1
metrics-util/tests/streaming/corpus/fe83f217d464f6fdfa5b2b1f87fe3a1a47371196 Normal file
View File

@ -0,0 +1 @@
<EFBFBD>

0
metrics-util/tests/streaming/crashes/.gitkeep Normal file
View File

9
metrics-util/tests/streaming/fuzz_target.rs Normal file
View File

@ -0,0 +1,9 @@
use bolero::fuzz;
use metrics_util::StreamingIntegers;
fn main() {
fuzz!().with_type().for_each(|value: &Vec<u64>| {
let mut si = StreamingIntegers::new();
si.compress(&value);
});
}