From 8e144c69e00d426248cc440328e0005e3ee96bd1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 23 Dec 2019 20:00:48 +0100 Subject: [PATCH] Start writing some low-level benchmarks --- .gitignore | 4 +- extras/benchmarks/bin/benchmarks.dart | 27 +++++++ .../benchmarks/lib/async_benchmark_base.dart | 53 +++++++++++++ extras/benchmarks/lib/benchmarks.dart | 77 +++++++++++++++++++ extras/benchmarks/lib/src/moor/database.dart | 0 .../lib/src/sqlite/bind_string.dart | 11 ++- .../lib/src/sqlparser/parse_moor_file.dart | 51 ++++++++++++ .../lib/src/sqlparser/tokenizer.dart | 35 +++++++++ extras/benchmarks/pubspec.yaml | 22 ++++++ .../test/comparing_emitter_test.dart | 37 +++++++++ moor_ffi/pubspec.yaml | 1 - moor_ffi/tool/benchmarks.dart | 11 --- 12 files changed, 315 insertions(+), 14 deletions(-) create mode 100644 extras/benchmarks/bin/benchmarks.dart create mode 100644 extras/benchmarks/lib/async_benchmark_base.dart create mode 100644 extras/benchmarks/lib/benchmarks.dart create mode 100644 extras/benchmarks/lib/src/moor/database.dart rename moor_ffi/tool/benchmark/select_string_variable.dart => extras/benchmarks/lib/src/sqlite/bind_string.dart (65%) create mode 100644 extras/benchmarks/lib/src/sqlparser/parse_moor_file.dart create mode 100644 extras/benchmarks/lib/src/sqlparser/tokenizer.dart create mode 100644 extras/benchmarks/pubspec.yaml create mode 100644 extras/benchmarks/test/comparing_emitter_test.dart delete mode 100644 moor_ffi/tool/benchmarks.dart diff --git a/.gitignore b/.gitignore index 0f1b4a47..0c72de13 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ lcov.info .packages pubspec.lock -.dart_tool/ \ No newline at end of file +.dart_tool/ + +benchmark_results.json \ No newline at end of file diff --git a/extras/benchmarks/bin/benchmarks.dart b/extras/benchmarks/bin/benchmarks.dart new file mode 100644 index 00000000..0e611dad --- /dev/null +++ b/extras/benchmarks/bin/benchmarks.dart @@ -0,0 +1,27 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:benchmarks/benchmarks.dart'; + +final File output = File('benchmark_results.json'); + +void main() { + final tracker = TrackingEmitter(); + ComparingEmitter comparer; + if (output.existsSync()) { + final content = json.decode(output.readAsStringSync()); + final oldData = (content as Map).cast(); + comparer = ComparingEmitter(oldData); + } else { + comparer = ComparingEmitter(); + } + + final emitter = MultiEmitter([tracker, comparer]); + final benchmarks = allBenchmarks(emitter); + + for (final benchmark in benchmarks) { + benchmark.report(); + } + + output.writeAsStringSync(json.encode(tracker.timings)); +} diff --git a/extras/benchmarks/lib/async_benchmark_base.dart b/extras/benchmarks/lib/async_benchmark_base.dart new file mode 100644 index 00000000..98f2790e --- /dev/null +++ b/extras/benchmarks/lib/async_benchmark_base.dart @@ -0,0 +1,53 @@ +part of 'benchmarks.dart'; + +class AsyncBenchmarkBase { + final String name; + final ScoreEmitter emitter; + + const AsyncBenchmarkBase(this.name, this.emitter); + + Future run() async {} + + Future warmup() { + return run(); + } + + Future exercise() { + return run(); + } + + Future setup() async {} + + Future teardown() async {} + + static Future measureFor( + Future Function() f, int minimumMillis) async { + final minimumMicros = minimumMillis * 1000; + var iter = 0; + final watch = Stopwatch(); + watch.start(); + var elapsed = 0; + while (elapsed < minimumMicros) { + await f(); + elapsed = watch.elapsedMicroseconds; + iter++; + } + return elapsed / iter; + } + + Future measure() async { + await setup(); + try { + // Warmup for at least 100ms. Discard result. + await measureFor(warmup, 100); + // Run the benchmark for at least 2000ms. + return await measureFor(exercise, 2000); + } finally { + await teardown(); + } + } + + Future report() async { + emitter.emit(name, await measure()); + } +} diff --git a/extras/benchmarks/lib/benchmarks.dart b/extras/benchmarks/lib/benchmarks.dart new file mode 100644 index 00000000..1780fc78 --- /dev/null +++ b/extras/benchmarks/lib/benchmarks.dart @@ -0,0 +1,77 @@ +import 'package:benchmark_harness/benchmark_harness.dart'; +import 'package:intl/intl.dart'; + +import 'src/sqlite/bind_string.dart'; +import 'src/sqlparser/parse_moor_file.dart'; +import 'src/sqlparser/tokenizer.dart'; + +part 'async_benchmark_base.dart'; + +List allBenchmarks(ScoreEmitter emitter) { + return [ + // low-level sqlite native interop + SelectStringBenchmark(emitter), + // sql parser + ParseMoorFile(emitter), + TokenizerBenchmark(emitter), + ]; +} + +class TrackingEmitter implements ScoreEmitter { + /// The average time it took to run each benchmark, in microseconds. + final Map timings = {}; + + @override + void emit(String testName, double value) { + timings[testName] = value; + } +} + +class ComparingEmitter implements ScoreEmitter { + final Map oldTimings; + + static final _percent = NumberFormat('##.##%'); + + ComparingEmitter([this.oldTimings = const {}]); + + @override + void emit(String testName, double value) { + final content = StringBuffer(testName) + ..write(': ') + ..write(value) + ..write(' us'); + + if (oldTimings.containsKey(testName)) { + final oldTime = oldTimings[testName]; + final increasedTime = value - oldTime; + + final relative = increasedTime.abs() / oldTime; + + content.write('; delta: '); + if (increasedTime < 0) { + content + ..write('$increasedTime us, -') + ..write(_percent.format(relative)); + } else { + content + ..write('+$increasedTime us, +') + ..write(_percent.format(relative)); + } + } + + print(content); + } +} + +class MultiEmitter implements ScoreEmitter { + final List delegates; + + const MultiEmitter(this.delegates); + + @override + void emit(String testName, double value) { + for (final delegate in delegates) { + delegate.emit(testName, value); + } + } +} diff --git a/extras/benchmarks/lib/src/moor/database.dart b/extras/benchmarks/lib/src/moor/database.dart new file mode 100644 index 00000000..e69de29b diff --git a/moor_ffi/tool/benchmark/select_string_variable.dart b/extras/benchmarks/lib/src/sqlite/bind_string.dart similarity index 65% rename from moor_ffi/tool/benchmark/select_string_variable.dart rename to extras/benchmarks/lib/src/sqlite/bind_string.dart index 2c872b38..520a1651 100644 --- a/moor_ffi/tool/benchmark/select_string_variable.dart +++ b/extras/benchmarks/lib/src/sqlite/bind_string.dart @@ -2,7 +2,8 @@ import 'package:benchmark_harness/benchmark_harness.dart'; import 'package:moor_ffi/database.dart'; class SelectStringBenchmark extends BenchmarkBase { - SelectStringBenchmark() : super('SELECT a string variable'); + SelectStringBenchmark(ScoreEmitter emitter) + : super('SELECTing a single string variable', emitter: emitter); PreparedStatement statement; Database database; @@ -18,6 +19,14 @@ class SelectStringBenchmark extends BenchmarkBase { statement.select(const ['hello sqlite, can you return this string?']); } + @override + void exercise() { + // repeat 1000 instead of 10 times to reduce variance + for (var i = 0; i < 1000; i++) { + run(); + } + } + @override void teardown() { statement.close(); diff --git a/extras/benchmarks/lib/src/sqlparser/parse_moor_file.dart b/extras/benchmarks/lib/src/sqlparser/parse_moor_file.dart new file mode 100644 index 00000000..bac1c449 --- /dev/null +++ b/extras/benchmarks/lib/src/sqlparser/parse_moor_file.dart @@ -0,0 +1,51 @@ +import 'package:benchmark_harness/benchmark_harness.dart'; + +import 'package:sqlparser/sqlparser.dart'; + +const file = ''' + +foo: SELECT + l_orderkey, + SUM(l_extendedprice * (1 - l_discount)) AS revenue, + o_orderdate, + o_shippriority + FROM + customer, + orders, + lineitem + WHERE + c_mktsegment = '%s' + and c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate < '%s' + and l_shipdate > '%s' + GROUP BY + l_orderkey, + o_orderdate, + o_shippriority + ORDER BY + revenue DESC, + o_orderdate; + +manyColumns: + SELECT a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z FROM test; +'''; + +class ParseMoorFile extends BenchmarkBase { + ParseMoorFile(ScoreEmitter emitter) + : super('Moor file: Parse only', emitter: emitter); + + final _engine = SqlEngine(useMoorExtensions: true); + + @override + void exercise() { + for (var i = 0; i < 10; i++) { + assert(_engine.parseMoorFile(file).errors.isEmpty); + } + } + + @override + void run() { + _engine.parseMoorFile(file); + } +} diff --git a/extras/benchmarks/lib/src/sqlparser/tokenizer.dart b/extras/benchmarks/lib/src/sqlparser/tokenizer.dart new file mode 100644 index 00000000..61ed263b --- /dev/null +++ b/extras/benchmarks/lib/src/sqlparser/tokenizer.dart @@ -0,0 +1,35 @@ +import 'dart:math'; + +import 'package:benchmark_harness/benchmark_harness.dart'; + +// ignore: implementation_imports +import 'package:sqlparser/src/reader/tokenizer/token.dart'; +// ignore: implementation_imports +import 'package:sqlparser/src/reader/tokenizer/scanner.dart'; + +class TokenizerBenchmark extends BenchmarkBase { + StringBuffer input; + + static const int size = 10000; + + TokenizerBenchmark(ScoreEmitter emitter) + : super('Tokenizing $size keywords', emitter: emitter); + + @override + void setup() { + input = StringBuffer(); + + final random = Random(); + final keywordLexemes = keywords.keys.toList(); + for (var i = 0; i < size; i++) { + final keyword = keywordLexemes[random.nextInt(keywordLexemes.length)]; + input..write(' ')..write(keyword); + } + } + + @override + void run() { + final scanner = Scanner(input.toString()); + scanner.scanTokens(); + } +} diff --git a/extras/benchmarks/pubspec.yaml b/extras/benchmarks/pubspec.yaml new file mode 100644 index 00000000..08284410 --- /dev/null +++ b/extras/benchmarks/pubspec.yaml @@ -0,0 +1,22 @@ +name: benchmarks +description: Runs simple and complex benchmarks to measure performance of moor and moor_ffi + +dependencies: + moor: + moor_ffi: + benchmark_harness: ^1.0.5 + intl: ^0.16.0 +dev_dependencies: + moor_generator: + build_runner: + test: + +dependency_overrides: + moor: + path: ../../moor + moor_ffi: + path: ../../moor_ffi + moor_generator: + path: ../../moor_generator + sqlparser: + path: ../../sqlparser \ No newline at end of file diff --git a/extras/benchmarks/test/comparing_emitter_test.dart b/extras/benchmarks/test/comparing_emitter_test.dart new file mode 100644 index 00000000..88c06cc9 --- /dev/null +++ b/extras/benchmarks/test/comparing_emitter_test.dart @@ -0,0 +1,37 @@ +import 'dart:async'; + +import 'package:benchmarks/benchmarks.dart'; +import 'package:test/test.dart'; + +void main() { + test('compares time for increase', () { + final comparer = ComparingEmitter({'foo': 10.0}); + final output = printsOf(() => comparer.emit('foo', 12.5)); + expect(output, ['foo: 12.5 us; delta: +2.5 us, +25%']); + }); + + test('compares time for decrease', () { + final comparer = ComparingEmitter({'foo': 10.0}); + final output = printsOf(() => comparer.emit('foo', 7.5)); + expect(output, ['foo: 7.5 us; delta: -2.5 us, -25%']); + }); + + test('no comparison when old value unknown', () { + final comparer = ComparingEmitter(); + final output = printsOf(() => comparer.emit('foo', 10)); + expect(output, ['foo: 10.0 us']); + }); +} + +List printsOf(Function() code) { + final output = []; + + runZoned( + code, + zoneSpecification: ZoneSpecification( + print: (_, __, ___, line) => output.add(line), + ), + ); + + return output; +} diff --git a/moor_ffi/pubspec.yaml b/moor_ffi/pubspec.yaml index 2483a241..74bef229 100644 --- a/moor_ffi/pubspec.yaml +++ b/moor_ffi/pubspec.yaml @@ -15,7 +15,6 @@ dependencies: dev_dependencies: test: ^1.6.0 path: ^1.6.0 - benchmark_harness: ^1.0.0 flutter: plugin: diff --git a/moor_ffi/tool/benchmarks.dart b/moor_ffi/tool/benchmarks.dart deleted file mode 100644 index 586ad774..00000000 --- a/moor_ffi/tool/benchmarks.dart +++ /dev/null @@ -1,11 +0,0 @@ -import 'benchmark/select_string_variable.dart'; - -void main() { - final benchmarks = [ - SelectStringBenchmark(), - ]; - - for (final benchmark in benchmarks) { - benchmark.report(); - } -}