Skip to content
This repository was archived by the owner on Feb 4, 2025. It is now read-only.

Commit 4c4adc1

Browse files
jakemac53jakemac53
authored
separate out benchmark into 4 stages: create, serialize, deserialize, hash (#154)
As mentioned in #153. This gives us a better overall picture of where time is going, in the various modes. Current results: ``` SdkMapsJsonWireBenchmark-create: 904ms SdkMapsBufferWireBenchmark-create: 711ms SdkMapsBuilderWireBenchmark-create: 713ms LazyMapsJsonWireBenchmark-create: 0ms LazyMapsBufferWireBenchmark-create: 0ms LazyWrappersBufferWireBenchmark-create: 420ms BuilderMapsJsonWireBenchmark-create: 374ms BuilderMapsBuilderWireBenchmark-create: 379ms SdkMapsJsonWireBenchmark-serialize: 746ms, 7177227 bytes SdkMapsBufferWireBenchmark-serialize: 633ms, 5122684 bytes SdkMapsBuilderWireBenchmark-serialize: 667ms, 4859868 bytes LazyMapsJsonWireBenchmark-serialize: 1104ms, 7177227 bytes LazyMapsBufferWireBenchmark-serialize: 543ms, 5122684 bytes LazyWrappersBufferWireBenchmark-serialize: 0ms, 2111761 bytes BuilderMapsJsonWireBenchmark-serialize: 1028ms, 7177227 bytes BuilderMapsBuilderWireBenchmark-serialize: 0ms, 2111761 bytes SdkMapsJsonWireBenchmark-deserialize: 1249ms SdkMapsBufferWireBenchmark-deserialize: 0ms SdkMapsBuilderWireBenchmark-deserialize: 0ms LazyMapsJsonWireBenchmark-deserialize: 1298ms LazyMapsBufferWireBenchmark-deserialize: 0ms LazyWrappersBufferWireBenchmark-deserialize: 0ms BuilderMapsJsonWireBenchmark-deserialize: 1257ms BuilderMapsBuilderWireBenchmark-deserialize: 0ms SdkMapsJsonWireBenchmark-process: 151ms, hash 23186292 SdkMapsBufferWireBenchmark-process: 249ms, hash 23186292 SdkMapsBuilderWireBenchmark-process: 541ms, hash 23186292 LazyMapsJsonWireBenchmark-process: 179ms, hash 23186292 LazyMapsBufferWireBenchmark-process: 251ms, hash 23186292 LazyWrappersBufferWireBenchmark-process: 367ms, hash 23186292 BuilderMapsJsonWireBenchmark-process: 207ms, hash 23186292 BuilderMapsBuilderWireBenchmark-process: 364ms, hash 23186292 SdkMapsJsonWireBenchmark-total: 3050ms SdkMapsBufferWireBenchmark-total: 1593ms SdkMapsBuilderWireBenchmark-total: 1921ms LazyMapsJsonWireBenchmark-total: 2581ms LazyMapsBufferWireBenchmark-total: 794ms LazyWrappersBufferWireBenchmark-total: 787ms BuilderMapsJsonWireBenchmark-total: 2866ms BuilderMapsBuilderWireBenchmark-total: 743ms ```
1 parent 587ff2a commit 4c4adc1

11 files changed

+188
-141
lines changed

‎pkgs/dart_model/benchmark/builder_maps_builder_wire_benchmark.dart‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:typed_data';
6+
57
import 'package:dart_model/src/json_buffer/json_buffer_builder.dart';
68

79
import 'serialization_benchmark.dart';
@@ -11,12 +13,12 @@ JsonBufferBuilder? runningBuffer;
1113
/// Benchmark accumulating data directly into a [JsonBufferBuilder].
1214
class BuilderMapsBuilderWireBenchmark extends SerializationBenchmark {
1315
@override
14-
void run() {
15-
createData();
16-
17-
serialized = runningBuffer!.serialize();
16+
Uint8List serialize(Map<String, Object?> data) {
17+
assert(data == runningBuffer!.map);
18+
return runningBuffer!.serialize();
1819
}
1920

21+
@override
2022
Map<String, Object?> createData() {
2123
final buffer = runningBuffer = JsonBufferBuilder();
2224
final map = buffer.map;
@@ -44,9 +46,8 @@ class BuilderMapsBuilderWireBenchmark extends SerializationBenchmark {
4446
}
4547

4648
@override
47-
void deserialize() {
48-
deserialized = JsonBufferBuilder.deserialize(serialized!).map;
49-
}
49+
Map<String, Object?> deserialize(Uint8List serialized) =>
50+
JsonBufferBuilder.deserialize(serialized).map;
5051

5152
Member _makeMember(String key) {
5253
final intKey = key.length;

‎pkgs/dart_model/benchmark/builder_maps_json_wire_benchmark.dart‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,10 @@ import 'builder_maps_builder_wire_benchmark.dart';
1111
/// serializing it to JSON.
1212
class BuilderMapsJsonWireBenchmark extends BuilderMapsBuilderWireBenchmark {
1313
@override
14-
void run() {
15-
serialized = json.fuse(utf8).encode(createData()) as Uint8List;
16-
}
14+
Uint8List serialize(Map<String, Object?> data) =>
15+
json.fuse(utf8).encode(data) as Uint8List;
1716

1817
@override
19-
void deserialize() {
20-
deserialized = json.fuse(utf8).decode(serialized!) as Map<String, Object?>;
21-
}
18+
Map<String, Object?> deserialize(Uint8List serialized) =>
19+
json.fuse(utf8).decode(serialized) as Map<String, Object?>;
2220
}

‎pkgs/dart_model/benchmark/lazy_maps_buffer_wire_benchmark.dart‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,18 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:typed_data';
6+
57
import 'json_buffer.dart';
68
import 'serialization_benchmark.dart';
79

810
/// Benchmark accumulating data into a [JsonBuffer] via [LazyMap].
911
class LazyMapsBufferWireBenchmark extends SerializationBenchmark {
1012
@override
11-
void run() {
12-
serialized = JsonBuffer(createData()).serialize();
13-
}
13+
Uint8List serialize(Map<String, Object?> data) =>
14+
JsonBuffer(data).serialize();
1415

16+
@override
1517
LazyMap createData() {
1618
return LazyMap(mapKeys, (key) {
1719
final intKey = int.parse(key);
@@ -44,9 +46,8 @@ class LazyMapsBufferWireBenchmark extends SerializationBenchmark {
4446
}
4547

4648
@override
47-
void deserialize() {
48-
deserialized = JsonBuffer.deserialize(serialized!).asMap;
49-
}
49+
Map<String, Object?> deserialize(Uint8List serialized) =>
50+
JsonBuffer.deserialize(serialized).asMap;
5051
}
5152

5253
/// An interface.

‎pkgs/dart_model/benchmark/lazy_maps_json_wire_benchmark.dart‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,10 @@ import 'lazy_maps_buffer_wire_benchmark.dart';
1111
/// serializing it to JSON.
1212
class LazyMapsJsonWireBenchmark extends LazyMapsBufferWireBenchmark {
1313
@override
14-
void run() {
15-
serialized = json.fuse(utf8).encode(createData()) as Uint8List;
16-
}
14+
Uint8List serialize(Map<String, Object?> data) =>
15+
json.fuse(utf8).encode(data) as Uint8List;
1716

1817
@override
19-
void deserialize() {
20-
deserialized = json.fuse(utf8).decode(serialized!) as Map<String, Object?>;
21-
}
18+
Map<String, Object?> deserialize(Uint8List serialized) =>
19+
json.fuse(utf8).decode(serialized) as Map<String, Object?>;
2220
}

‎pkgs/dart_model/benchmark/lazy_wrappers_buffer_wire_benchmark.dart‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// BSD-style license that can be found in the LICENSE file.
44

55
import 'dart:collection';
6+
import 'dart:typed_data';
67

78
import 'package:dart_model/src/json_buffer/json_buffer_builder.dart';
89

@@ -15,12 +16,12 @@ JsonBufferBuilder? runningBuffer;
1516
/// extension type).
1617
class LazyWrappersBufferWireBenchmark extends SerializationBenchmark {
1718
@override
18-
void run() {
19-
createData();
20-
21-
serialized = runningBuffer!.serialize();
19+
Uint8List serialize(Map<String, Object?> data) {
20+
assert(data == runningBuffer!.map);
21+
return runningBuffer!.serialize();
2222
}
2323

24+
@override
2425
Map<String, Object?> createData() {
2526
final buffer = runningBuffer = JsonBufferBuilder();
2627
final map = buffer.map;
@@ -62,13 +63,12 @@ class LazyWrappersBufferWireBenchmark extends SerializationBenchmark {
6263
}
6364

6465
@override
65-
void deserialize() {
66-
deserialized = _LazyMap<Object?, Interface>(
67-
JsonBufferBuilder.deserialize(serialized!).map,
68-
(json) => Interface.fromJson(json as Map<String, Object?>),
69-
(i) => i.toJson(),
70-
);
71-
}
66+
Map<String, Object?> deserialize(Uint8List serialized) =>
67+
_LazyMap<Object?, Interface>(
68+
JsonBufferBuilder.deserialize(serialized).map,
69+
(json) => Interface.fromJson(json as Map<String, Object?>),
70+
(i) => i.toJson(),
71+
);
7272
}
7373

7474
class _LazyMap<From, To> extends MapBase<String, To> {

‎pkgs/dart_model/benchmark/main.dart‎

Lines changed: 56 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:math';
6+
57
import 'package:collection/collection.dart';
68

79
import 'builder_maps_builder_wire_benchmark.dart';
@@ -15,51 +17,76 @@ import 'regular_dart_classes.dart' as regular;
1517
import 'sdk_maps_buffer_wire_benchmark.dart';
1618
import 'sdk_maps_builder_wire_benchmark.dart';
1719
import 'sdk_maps_json_wire_benchmark.dart';
20+
import 'serialization_benchmark.dart';
1821

1922
void main() {
20-
final sdkMapsJsonWireBenchmark = SdkMapsJsonWireBenchmark();
21-
final lazyMapsBufferWireBenchmark = LazyMapsBufferWireBenchmark();
22-
final lazyWrappersBufferWireBenchmark = LazyWrappersBufferWireBenchmark();
23-
final builderMapsBuilderWireBenchmark = BuilderMapsBuilderWireBenchmark();
24-
final regularClassesBufferWireBenchmark = RegularClassesBufferWireBenchmark();
2523
final serializationBenchmarks = [
26-
sdkMapsJsonWireBenchmark,
24+
SdkMapsJsonWireBenchmark(),
2725
SdkMapsBufferWireBenchmark(),
2826
SdkMapsBuilderWireBenchmark(),
2927
LazyMapsJsonWireBenchmark(),
30-
lazyMapsBufferWireBenchmark,
31-
lazyWrappersBufferWireBenchmark,
28+
LazyMapsBufferWireBenchmark(),
29+
LazyWrappersBufferWireBenchmark(),
3230
BuilderMapsJsonWireBenchmark(),
33-
builderMapsBuilderWireBenchmark,
34-
regularClassesBufferWireBenchmark,
31+
BuilderMapsBuilderWireBenchmark(),
32+
RegularClassesBufferWireBenchmark(),
3533
];
3634

3735
for (var i = 0; i != 3; ++i) {
38-
for (final benchmark in serializationBenchmarks) {
39-
final measure = benchmark.measure().toMilliseconds;
40-
final paddedName = benchmark.name.padLeft(31);
41-
final paddedMeasure = '${measure}ms'.padLeft(6);
36+
// Collects the total measurements from all phases, per benchmark.
37+
final totals = <SerializationBenchmark, int>{
38+
for (var benchmark in serializationBenchmarks) benchmark: 0,
39+
};
40+
41+
for (var stage in BenchmarkStage.values) {
42+
var padding = 0;
43+
for (final benchmark in serializationBenchmarks) {
44+
benchmark.stage = stage;
45+
padding = max(padding, benchmark.name.length);
46+
}
4247

43-
final paddedBytes = '${benchmark.serialized!.length}'.padLeft(7);
44-
print('$paddedName: $paddedMeasure, $paddedBytes bytes');
45-
benchmark.deserialize();
48+
for (final benchmark in serializationBenchmarks) {
49+
final measure = benchmark.measure().toMilliseconds;
50+
totals[benchmark] = totals[benchmark]! + measure;
51+
52+
var buffer =
53+
StringBuffer(benchmark.name.padLeft(padding + 1))
54+
..write(': ')
55+
..write('${measure}ms'.padLeft(6));
56+
57+
switch (stage) {
58+
case BenchmarkStage.serialize:
59+
final paddedBytes = '${benchmark.bytes}'.padLeft(7);
60+
buffer.write(', $paddedBytes bytes');
61+
case BenchmarkStage.process:
62+
buffer.write(', hash ${benchmark.hashResult}');
63+
default:
64+
}
65+
print(buffer.toString());
66+
}
4667
}
47-
print('');
4868

49-
for (final benchmark in [
50-
sdkMapsJsonWireBenchmark.processBenchmark(),
51-
lazyMapsBufferWireBenchmark.processBenchmark(),
52-
lazyWrappersBufferWireBenchmark.processBenchmark(),
53-
builderMapsBuilderWireBenchmark.processBenchmark(),
54-
regularClassesBufferWireBenchmark.processBenchmark(),
55-
]) {
56-
final measure = benchmark.measure().toMilliseconds;
57-
final paddedName = benchmark.name.padLeft(36);
58-
final paddedMeasure = '${measure}ms'.padLeft(6);
69+
// Add up the totals and print them.
70+
{
71+
var padding = 0;
72+
String name(SerializationBenchmark benchmark) =>
73+
'${benchmark.runtimeType}-total';
5974

60-
print('$paddedName: $paddedMeasure, hash ${benchmark.computedResult}');
75+
for (final benchmark in serializationBenchmarks) {
76+
padding = max(padding, name(benchmark).length);
77+
}
78+
for (var benchmark in serializationBenchmarks) {
79+
var buffer =
80+
StringBuffer(name(benchmark).padLeft(padding + 1))
81+
..write(':')
82+
..write('${totals[benchmark]}ms'.padLeft(7));
83+
print(buffer.toString());
84+
}
6185
}
6286

87+
print('');
88+
89+
print('validating benchmark results (this is slow)');
6390
for (final benchmark in serializationBenchmarks.skip(1)) {
6491
var deserialized = benchmark.deserialized;
6592
// Need to unwrap these to compare them as raw maps.

‎pkgs/dart_model/benchmark/regular_dart_classes.dart‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,18 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:typed_data';
6+
57
import 'package:dart_model/src/json_buffer/json_buffer_builder.dart';
68

79
import 'serialization_benchmark.dart';
810

911
JsonBufferBuilder? runningBuffer;
1012

11-
/// Benchmark accumulating data directly into a [JsonBufferBuilder] with an
12-
/// indirection through a thin wrapper type (which is a real type, not an
13-
/// extension type).
13+
/// Benchmark for regular dart classes which serialize and deserializ into
14+
/// a [JsonBufferBuilder].
1415
class RegularClassesBufferWireBenchmark extends SerializationBenchmark {
1516
@override
16-
void run() {
17-
var data = createData();
18-
19-
final buffer = runningBuffer = JsonBufferBuilder();
20-
data.forEach((k, v) => buffer.map[k] = v.toJson());
21-
serialized = runningBuffer!.serialize();
22-
}
23-
2417
/// Creates the data, but its not ready yet to be serialized.
2518
Map<String, Interface> createData() {
2619
final map = <String, Interface>{};
@@ -62,12 +55,16 @@ class RegularClassesBufferWireBenchmark extends SerializationBenchmark {
6255
}
6356

6457
@override
65-
void deserialize() {
66-
deserialized = JsonBufferBuilder.deserialize(
67-
serialized!,
68-
).map.map<String, Interface>(
69-
(k, v) => MapEntry(k, Interface.fromJson(v as Map<String, Object?>)),
70-
);
58+
Map<String, Object?> deserialize(Uint8List serialized) =>
59+
JsonBufferBuilder.deserialize(serialized).map.map<String, Interface>(
60+
(k, v) => MapEntry(k, Interface.fromJson(v as Map<String, Object?>)),
61+
);
62+
63+
@override
64+
Uint8List serialize(Map<String, Object?> data) {
65+
final buffer = runningBuffer = JsonBufferBuilder();
66+
data.forEach((k, v) => buffer.map[k] = (v as Interface).toJson());
67+
return runningBuffer!.serialize();
7168
}
7269
}
7370

‎pkgs/dart_model/benchmark/sdk_maps_buffer_wire_benchmark.dart‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,19 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:typed_data';
6+
57
import 'json_buffer.dart';
68
import 'sdk_maps_json_wire_benchmark.dart';
79

810
/// Benchmark accumulating data into SDK maps then serializing it via
911
/// [JsonBuffer].
1012
class SdkMapsBufferWireBenchmark extends SdkMapsJsonWireBenchmark {
1113
@override
12-
void run() {
13-
serialized = JsonBuffer(createData()).serialize();
14-
}
14+
Uint8List serialize(Map<String, Object?> data) =>
15+
JsonBuffer(data).serialize();
1516

1617
@override
17-
void deserialize() {
18-
deserialized = JsonBuffer.deserialize(serialized!).asMap;
19-
}
18+
Map<String, Object?> deserialize(Uint8List serialized) =>
19+
JsonBuffer.deserialize(serialized).asMap;
2020
}

‎pkgs/dart_model/benchmark/sdk_maps_builder_wire_benchmark.dart‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:typed_data';
6+
57
import 'package:dart_model/src/json_buffer/json_buffer_builder.dart';
68

79
import 'sdk_maps_json_wire_benchmark.dart';
@@ -10,14 +12,13 @@ import 'sdk_maps_json_wire_benchmark.dart';
1012
/// [JsonBufferBuilder].
1113
class SdkMapsBuilderWireBenchmark extends SdkMapsJsonWireBenchmark {
1214
@override
13-
void run() {
15+
Uint8List serialize(Map<String, Object?> data) {
1416
final builder = JsonBufferBuilder();
15-
builder.map.addAll(createData());
16-
serialized = builder.serialize();
17+
builder.map.addAll(data);
18+
return builder.serialize();
1719
}
1820

1921
@override
20-
void deserialize() {
21-
deserialized = JsonBufferBuilder.deserialize(serialized!).map;
22-
}
22+
Map<String, Object?> deserialize(Uint8List serialized) =>
23+
JsonBufferBuilder.deserialize(serialized).map;
2324
}

‎pkgs/dart_model/benchmark/sdk_maps_json_wire_benchmark.dart‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ import 'serialization_benchmark.dart';
1010
/// Benchmark accumulating data into SDK maps then serializing it to JSON.
1111
class SdkMapsJsonWireBenchmark extends SerializationBenchmark {
1212
@override
13-
void run() {
14-
serialized = json.fuse(utf8).encode(createData()) as Uint8List;
15-
}
13+
Uint8List serialize(Map<String, Object?> data) =>
14+
json.fuse(utf8).encode(data) as Uint8List;
1615

16+
@override
1717
Map<String, Object?> createData() {
1818
return Map<String, Object?>.fromIterable(
1919
mapKeys,
@@ -53,8 +53,8 @@ class SdkMapsJsonWireBenchmark extends SerializationBenchmark {
5353
}
5454

5555
@override
56-
void deserialize() {
57-
deserialized = json.fuse(utf8).decode(serialized!) as Map<String, Object?>;
56+
Map<String, Object?> deserialize(Uint8List serialized) {
57+
return json.fuse(utf8).decode(serialized) as Map<String, Object?>;
5858
}
5959
}
6060

0 commit comments

Comments
 (0)