Start writing some low-level benchmarks

.gitignore

@@ -5,4 +5,6 @@ lcov.info
 
 .packages
 pubspec.lock
-.dart_tool/
+.dart_tool/
+
+benchmark_results.json

extras/benchmarks/bin/benchmarks.dart

@@ -0,0 +1,27 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:benchmarks/benchmarks.dart';
+
+final File output = File('benchmark_results.json');
+
+void main() {
+  final tracker = TrackingEmitter();
+  ComparingEmitter comparer;
+  if (output.existsSync()) {
+    final content = json.decode(output.readAsStringSync());
+    final oldData = (content as Map).cast<String, double>();
+    comparer = ComparingEmitter(oldData);
+  } else {
+    comparer = ComparingEmitter();
+  }
+
+  final emitter = MultiEmitter([tracker, comparer]);
+  final benchmarks = allBenchmarks(emitter);
+
+  for (final benchmark in benchmarks) {
+    benchmark.report();
+  }
+
+  output.writeAsStringSync(json.encode(tracker.timings));
+}

extras/benchmarks/lib/async_benchmark_base.dart

@@ -0,0 +1,53 @@
+part of 'benchmarks.dart';
+
+class AsyncBenchmarkBase {
+  final String name;
+  final ScoreEmitter emitter;
+
+  const AsyncBenchmarkBase(this.name, this.emitter);
+
+  Future<void> run() async {}
+
+  Future<void> warmup() {
+    return run();
+  }
+
+  Future<void> exercise() {
+    return run();
+  }
+
+  Future<void> setup() async {}
+
+  Future<void> teardown() async {}
+
+  static Future<double> measureFor(
+      Future Function() f, int minimumMillis) async {
+    final minimumMicros = minimumMillis * 1000;
+    var iter = 0;
+    final watch = Stopwatch();
+    watch.start();
+    var elapsed = 0;
+    while (elapsed < minimumMicros) {
+      await f();
+      elapsed = watch.elapsedMicroseconds;
+      iter++;
+    }
+    return elapsed / iter;
+  }
+
+  Future<double> measure() async {
+    await setup();
+    try {
+      // Warmup for at least 100ms. Discard result.
+      await measureFor(warmup, 100);
+      // Run the benchmark for at least 2000ms.
+      return await measureFor(exercise, 2000);
+    } finally {
+      await teardown();
+    }
+  }
+
+  Future<void> report() async {
+    emitter.emit(name, await measure());
+  }
+}

extras/benchmarks/lib/benchmarks.dart

@@ -0,0 +1,77 @@
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:intl/intl.dart';
+
+import 'src/sqlite/bind_string.dart';
+import 'src/sqlparser/parse_moor_file.dart';
+import 'src/sqlparser/tokenizer.dart';
+
+part 'async_benchmark_base.dart';
+
+List<BenchmarkBase> allBenchmarks(ScoreEmitter emitter) {
+  return [
+    // low-level sqlite native interop
+    SelectStringBenchmark(emitter),
+    // sql parser
+    ParseMoorFile(emitter),
+    TokenizerBenchmark(emitter),
+  ];
+}
+
+class TrackingEmitter implements ScoreEmitter {
+  /// The average time it took to run each benchmark, in microseconds.
+  final Map<String, double> timings = {};
+
+  @override
+  void emit(String testName, double value) {
+    timings[testName] = value;
+  }
+}
+
+class ComparingEmitter implements ScoreEmitter {
+  final Map<String, double> oldTimings;
+
+  static final _percent = NumberFormat('##.##%');
+
+  ComparingEmitter([this.oldTimings = const {}]);
+
+  @override
+  void emit(String testName, double value) {
+    final content = StringBuffer(testName)
+      ..write(': ')
+      ..write(value)
+      ..write(' us');
+
+    if (oldTimings.containsKey(testName)) {
+      final oldTime = oldTimings[testName];
+      final increasedTime = value - oldTime;
+
+      final relative = increasedTime.abs() / oldTime;
+
+      content.write('; delta: ');
+      if (increasedTime < 0) {
+        content
+          ..write('$increasedTime us, -')
+          ..write(_percent.format(relative));
+      } else {
+        content
+          ..write('+$increasedTime us, +')
+          ..write(_percent.format(relative));
+      }
+    }
+
+    print(content);
+  }
+}
+
+class MultiEmitter implements ScoreEmitter {
+  final List<ScoreEmitter> delegates;
+
+  const MultiEmitter(this.delegates);
+
+  @override
+  void emit(String testName, double value) {
+    for (final delegate in delegates) {
+      delegate.emit(testName, value);
+    }
+  }
+}

extras/benchmarks/lib/src/sqlite/bind_string.dart

@@ -2,7 +2,8 @@ import 'package:benchmark_harness/benchmark_harness.dart';
 import 'package:moor_ffi/database.dart';
 
 class SelectStringBenchmark extends BenchmarkBase {
-  SelectStringBenchmark() : super('SELECT a string variable');
+  SelectStringBenchmark(ScoreEmitter emitter)
+      : super('SELECTing a single string variable', emitter: emitter);
 
   PreparedStatement statement;
   Database database;
@@ -18,6 +19,14 @@ class SelectStringBenchmark extends BenchmarkBase {
     statement.select(const ['hello sqlite, can you return this string?']);
   }
 
+  @override
+  void exercise() {
+    // repeat 1000 instead of 10 times to reduce variance
+    for (var i = 0; i < 1000; i++) {
+      run();
+    }
+  }
+
   @override
   void teardown() {
     statement.close();

extras/benchmarks/lib/src/sqlparser/parse_moor_file.dart

@@ -0,0 +1,51 @@
+import 'package:benchmark_harness/benchmark_harness.dart';
+
+import 'package:sqlparser/sqlparser.dart';
+
+const file = '''
+
+foo: SELECT
+    l_orderkey,
+    SUM(l_extendedprice * (1 - l_discount)) AS revenue,
+    o_orderdate,
+    o_shippriority
+  FROM
+    customer,
+    orders,
+    lineitem
+  WHERE
+    c_mktsegment = '%s'
+    and c_custkey = o_custkey
+    and l_orderkey = o_orderkey
+    and o_orderdate < '%s'
+    and l_shipdate > '%s'
+  GROUP BY
+    l_orderkey,
+    o_orderdate,
+    o_shippriority
+  ORDER BY
+    revenue DESC,
+    o_orderdate;
+    
+manyColumns: 
+  SELECT a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z FROM test;
+''';
+
+class ParseMoorFile extends BenchmarkBase {
+  ParseMoorFile(ScoreEmitter emitter)
+      : super('Moor file: Parse only', emitter: emitter);
+
+  final _engine = SqlEngine(useMoorExtensions: true);
+
+  @override
+  void exercise() {
+    for (var i = 0; i < 10; i++) {
+      assert(_engine.parseMoorFile(file).errors.isEmpty);
+    }
+  }
+
+  @override
+  void run() {
+    _engine.parseMoorFile(file);
+  }
+}

extras/benchmarks/lib/src/sqlparser/tokenizer.dart

@@ -0,0 +1,35 @@
+import 'dart:math';
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+
+// ignore: implementation_imports
+import 'package:sqlparser/src/reader/tokenizer/token.dart';
+// ignore: implementation_imports
+import 'package:sqlparser/src/reader/tokenizer/scanner.dart';
+
+class TokenizerBenchmark extends BenchmarkBase {
+  StringBuffer input;
+
+  static const int size = 10000;
+
+  TokenizerBenchmark(ScoreEmitter emitter)
+      : super('Tokenizing $size keywords', emitter: emitter);
+
+  @override
+  void setup() {
+    input = StringBuffer();
+
+    final random = Random();
+    final keywordLexemes = keywords.keys.toList();
+    for (var i = 0; i < size; i++) {
+      final keyword = keywordLexemes[random.nextInt(keywordLexemes.length)];
+      input..write(' ')..write(keyword);
+    }
+  }
+
+  @override
+  void run() {
+    final scanner = Scanner(input.toString());
+    scanner.scanTokens();
+  }
+}

extras/benchmarks/pubspec.yaml

@@ -0,0 +1,22 @@
+name: benchmarks
+description: Runs simple and complex benchmarks to measure performance of moor and moor_ffi
+
+dependencies:
+  moor:
+  moor_ffi:
+  benchmark_harness: ^1.0.5
+  intl: ^0.16.0
+dev_dependencies:
+  moor_generator:
+  build_runner:
+  test:
+
+dependency_overrides:
+  moor:
+    path: ../../moor
+  moor_ffi:
+    path: ../../moor_ffi
+  moor_generator:
+    path: ../../moor_generator
+  sqlparser:
+    path: ../../sqlparser

extras/benchmarks/test/comparing_emitter_test.dart

@@ -0,0 +1,37 @@
+import 'dart:async';
+
+import 'package:benchmarks/benchmarks.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test('compares time for increase', () {
+    final comparer = ComparingEmitter({'foo': 10.0});
+    final output = printsOf(() => comparer.emit('foo', 12.5));
+    expect(output, ['foo: 12.5 us; delta: +2.5 us, +25%']);
+  });
+
+  test('compares time for decrease', () {
+    final comparer = ComparingEmitter({'foo': 10.0});
+    final output = printsOf(() => comparer.emit('foo', 7.5));
+    expect(output, ['foo: 7.5 us; delta: -2.5 us, -25%']);
+  });
+
+  test('no comparison when old value unknown', () {
+    final comparer = ComparingEmitter();
+    final output = printsOf(() => comparer.emit('foo', 10));
+    expect(output, ['foo: 10.0 us']);
+  });
+}
+
+List<String> printsOf(Function() code) {
+  final output = <String>[];
+
+  runZoned(
+    code,
+    zoneSpecification: ZoneSpecification(
+      print: (_, __, ___, line) => output.add(line),
+    ),
+  );
+
+  return output;
+}

moor_ffi/pubspec.yaml

@@ -15,7 +15,6 @@ dependencies:
 dev_dependencies:
   test: ^1.6.0
   path: ^1.6.0
-  benchmark_harness: ^1.0.0
 
 flutter:
   plugin:

moor_ffi/tool/benchmarks.dart

@@ -1,11 +0,0 @@
-import 'benchmark/select_string_variable.dart';
-
-void main() {
-  final benchmarks = [
-    SelectStringBenchmark(),
-  ];
-
-  for (final benchmark in benchmarks) {
-    benchmark.report();
-  }
-}