From b579b1fb906a45392b2e86ffce6a2e640559b0ae Mon Sep 17 00:00:00 2001 From: Grieshofer Philipp Date: Thu, 14 Jul 2022 16:24:22 +0200 Subject: [PATCH] Implement in-memory storage backend --- hive/lib/hive.dart | 17 ++- .../src/backend/storage_backend_memory.dart | 73 ------------ .../backend/stub/random_access_buffer.dart | 100 ++++++++++++++++ .../backend/stub/storage_backend_memory.dart | 79 +++++++++++++ .../memory/random_access_buffer_test.dart | 73 ++++++++++++ .../memory/storage_backend_memory_test.dart | 111 ++++++++++++++++++ .../backend/storage_backend_memory_test.dart | 64 ---------- 7 files changed, 379 insertions(+), 138 deletions(-) delete mode 100644 hive/lib/src/backend/storage_backend_memory.dart create mode 100644 hive/lib/src/backend/stub/random_access_buffer.dart create mode 100644 hive/lib/src/backend/stub/storage_backend_memory.dart create mode 100644 hive/test/tests/backend/memory/random_access_buffer_test.dart create mode 100644 hive/test/tests/backend/memory/storage_backend_memory_test.dart delete mode 100644 hive/test/tests/backend/storage_backend_memory_test.dart diff --git a/hive/lib/hive.dart b/hive/lib/hive.dart index 11070d664..e26ac8c9a 100644 --- a/hive/lib/hive.dart +++ b/hive/lib/hive.dart @@ -26,24 +26,39 @@ export 'package:hive/src/box_collection/box_collection.dart'; export 'src/backend/js/web_worker/web_worker_stub.dart' if (dart.library.html) 'src/backend/js/web_worker/web_worker.dart'; -export 'src/backend/storage_backend_memory.dart'; +export 'src/backend/stub/storage_backend_memory.dart'; export 'src/object/hive_object.dart' show HiveObject, HiveObjectMixin; part 'src/annotations/hive_field.dart'; + part 'src/annotations/hive_type.dart'; + part 'src/binary/binary_reader.dart'; + part 'src/binary/binary_writer.dart'; + part 'src/box/box.dart'; + part 'src/box/box_base.dart'; + part 'src/box/lazy_box.dart'; + part 'src/crypto/hive_aes_cipher.dart'; + part 'src/crypto/hive_cipher.dart'; + part 'src/hive.dart'; + part 'src/hive_error.dart'; + part 'src/object/hive_collection.dart'; + part 'src/object/hive_list.dart'; + part 'src/object/hive_storage_backend_preference.dart'; + part 'src/registry/type_adapter.dart'; + part 'src/registry/type_registry.dart'; /// Global constant to access Hive. diff --git a/hive/lib/src/backend/storage_backend_memory.dart b/hive/lib/src/backend/storage_backend_memory.dart deleted file mode 100644 index 2cbaec6d7..000000000 --- a/hive/lib/src/backend/storage_backend_memory.dart +++ /dev/null @@ -1,73 +0,0 @@ -import 'dart:typed_data'; - -import 'package:hive/hive.dart'; -import 'package:hive/src/backend/storage_backend.dart'; -import 'package:hive/src/binary/frame.dart'; -import 'package:hive/src/binary/frame_helper.dart'; -import 'package:hive/src/box/keystore.dart'; - -/// In-memory Storage backend -class StorageBackendMemory extends StorageBackend { - final HiveCipher? _cipher; - - final FrameHelper _frameHelper; - - Uint8List? _bytes; - - /// Not part of public API - StorageBackendMemory(Uint8List? bytes, this._cipher) - : _bytes = bytes ?? Uint8List(0), - _frameHelper = FrameHelper(); - - @override - String? get path => null; - - @override - bool supportsCompaction = false; - - @override - Future initialize( - TypeRegistry registry, Keystore? keystore, bool lazy) async { - var recoveryOffset = await _frameHelper.framesFromBytes( - _bytes!, // Initialized at constructor and nulled after initialization - keystore, - registry, - _cipher, - ); - - if (recoveryOffset != -1) { - throw HiveError('Wrong checksum in bytes. Box may be corrupted.'); - } - - _bytes = null; - - return Future.value(); - } - - @override - Future readValue(Frame frame) { - throw UnsupportedError('This operation is unsupported for memory boxes.'); - } - - @override - Future writeFrames(List frames) => Future.value(); - - @override - Future> compact(Iterable frames) { - throw UnsupportedError('This operation is unsupported for memory boxes.'); - } - - @override - Future clear() => Future.value(); - - @override - Future close() => Future.value(); - - @override - Future deleteFromDisk() { - throw UnsupportedError('This operation is unsupported for memory boxes.'); - } - - @override - Future flush() => Future.value(); -} diff --git a/hive/lib/src/backend/stub/random_access_buffer.dart b/hive/lib/src/backend/stub/random_access_buffer.dart new file mode 100644 index 000000000..d4dd25f40 --- /dev/null +++ b/hive/lib/src/backend/stub/random_access_buffer.dart @@ -0,0 +1,100 @@ +import 'dart:typed_data'; + +import 'package:hive/hive.dart'; +import 'package:hive/src/backend/vm/read_write_sync.dart'; +import 'package:hive/src/binary/binary_reader_impl.dart'; +import 'package:hive/src/binary/binary_writer_impl.dart'; +import 'package:hive/src/binary/frame.dart'; +import 'package:hive/src/binary/frame_helper.dart'; +import 'package:hive/src/box/keystore.dart'; + +/// Implementation of [RandomAccessBuffer] +/// +/// Provides simplified access to raw [Uint8List] for storing +/// [Frame] data. For main usage in [StorageBackendMemory]. +class RandomAccessBuffer { + final ReadWriteSync _sync; + + final FrameHelper _frameHelper; + + Uint8List _bytes; + + int _writeOffset; + + RandomAccessBuffer(Uint8List? bytes, {int initialOffset = 0}) + : _bytes = bytes ?? Uint8List(0), + _sync = ReadWriteSync(), + _writeOffset = initialOffset, + _frameHelper = FrameHelper(); + + /// Get the current length in bytes + int get length => _bytes.lengthInBytes; + + /// Get the current write offset + int get writeOffset => _writeOffset; + + /// Read a [Frame] from the [RandomAccessBuffer] + Future read( + Frame frame, TypeRegistry typeRegistry, HiveCipher? cipher) { + return _sync.syncRead(() async { + var bytes = _bytes.sublist(frame.offset, frame.offset + frame.length!); + var reader = BinaryReaderImpl(bytes, typeRegistry); + var readFrame = await reader.readFrame(cipher: cipher, lazy: false); + + if (readFrame == null) { + throw HiveError('Could not read value from box. ' + 'Maybe your box is corrupted.'); + } + + return readFrame.value; + }); + } + + /// Write a list of [Frame] to the [RandomAccessBuffer] + Future write( + List frames, TypeRegistry typeRegistry, HiveCipher? cipher) { + return _sync.syncWrite(() async { + var writer = BinaryWriterImpl(typeRegistry); + + for (var frame in frames) { + frame.length = await writer.writeFrame(frame, cipher: cipher); + } + + var b = BytesBuilder(); + b.add(_bytes); + b.add(writer.toBytes()); + _bytes = b.toBytes(); + + for (var frame in frames) { + frame.offset = _writeOffset; + _writeOffset += frame.length!; + } + }); + } + + /// Perform the recovery check + /// Throws an [HiveError] on box corruption + Future recoveryCheck( + TypeRegistry typeRegistry, Keystore? keystore, HiveCipher? cipher) async { + var recoveryOffset = await _frameHelper.framesFromBytes( + _bytes, + keystore, + typeRegistry, + cipher, + ); + + if (recoveryOffset != -1) { + throw HiveError('Wrong checksum in bytes. Box may be corrupted.'); + } + + _writeOffset = _bytes.offsetInBytes; + } + + /// Clear the buffer and reset the write offset + Future clear() async { + return _sync.syncReadWrite(() async { + _bytes = Uint8List(0); + _writeOffset = 0; + }); + } +} diff --git a/hive/lib/src/backend/stub/storage_backend_memory.dart b/hive/lib/src/backend/stub/storage_backend_memory.dart new file mode 100644 index 000000000..7ea77f3d4 --- /dev/null +++ b/hive/lib/src/backend/stub/storage_backend_memory.dart @@ -0,0 +1,79 @@ +import 'dart:typed_data'; + +import 'package:hive/hive.dart'; +import 'package:hive/src/backend/storage_backend.dart'; +import 'package:hive/src/backend/stub/random_access_buffer.dart'; +import 'package:hive/src/binary/frame.dart'; +import 'package:hive/src/box/keystore.dart'; + +/// In-memory storage backend +/// +/// This storage memory uses the [RandomAccessBuffer] which is an abstraction +/// for managing data in [Uint8List] as a mean of storage. Data will not be +/// persisted and is lost on close. Data is not stored, is highly volatile +/// and does not survive a page refresh. +class StorageBackendMemory extends StorageBackend { + late final RandomAccessBuffer _randomAccessBuffer; + + final HiveCipher? _cipher; + + TypeRegistry? _typeRegistry; + + /// Default constructor for the [StorageBackendMemory] + /// + /// If needed, provide a [Uint8List] with already existing bytes and the + /// corresponding [initialOffset]. For encryption, provide a [_cipher]. + StorageBackendMemory(Uint8List? bytes, this._cipher, + [int initialOffset = 0]) { + _randomAccessBuffer = + RandomAccessBuffer(bytes, initialOffset: initialOffset); + } + + /// An in-memory storage has no path + @override + final path = null; + + @override + final bool supportsCompaction = false; + + @override + Future initialize( + TypeRegistry registry, Keystore keystore, bool lazy) async { + _typeRegistry = registry; + await _randomAccessBuffer.recoveryCheck(registry, keystore, _cipher); + } + + @override + Future readValue(Frame frame) async { + return _randomAccessBuffer.read(frame, _typeRegistry!, _cipher); + } + + @override + Future writeFrames(List frames) { + return _randomAccessBuffer.write(frames, _typeRegistry!, _cipher); + } + + @override + Future compact(Iterable frames) { + throw UnsupportedError('This operation is unsupported for memory boxes.'); + } + + @override + Future clear() { + return _randomAccessBuffer.clear(); + } + + @override + Future close() { + return _randomAccessBuffer.clear(); + } + + @override + Future deleteFromDisk() { + return _randomAccessBuffer.clear(); + } + + /// Nothing to flush to disk as we act directly in-memory + @override + Future flush() => Future.value(); +} diff --git a/hive/test/tests/backend/memory/random_access_buffer_test.dart b/hive/test/tests/backend/memory/random_access_buffer_test.dart new file mode 100644 index 000000000..32591190a --- /dev/null +++ b/hive/test/tests/backend/memory/random_access_buffer_test.dart @@ -0,0 +1,73 @@ +import 'package:hive/src/backend/stub/random_access_buffer.dart'; +import 'package:hive/src/binary/binary_writer_impl.dart'; +import 'package:hive/src/binary/frame.dart'; +import 'package:hive/src/registry/type_registry_impl.dart'; +import 'package:test/test.dart'; + + +void main() { + group('RandomAccessBuffer', () { + test('empty random access buffer', () { + final randomAccessBuffer = RandomAccessBuffer(null); + expect(randomAccessBuffer.length, 0); + expect(randomAccessBuffer.writeOffset, 0); + }); + + test('write() writes to random access buffer', () async { + final typeRegistry = TypeRegistryImpl.nullImpl; + + final randomAccessBuffer = RandomAccessBuffer(null); + randomAccessBuffer.recoveryCheck(typeRegistry, null, null); + + final frame = Frame("abc", "123"); + await randomAccessBuffer.write([frame], typeRegistry, null); + + expect(randomAccessBuffer.length, getFrameBytes([frame])); + expect(randomAccessBuffer.writeOffset, getFrameBytes([frame])); + }); + + test('read() reads data from random access buffer', () async { + final typeRegistry = TypeRegistryImpl.nullImpl; + + final randomAccessBuffer = RandomAccessBuffer(null); + randomAccessBuffer.recoveryCheck(typeRegistry, null, null); + + final saveFrame = Frame("abc", "123", offset: 0, length: 3); + await randomAccessBuffer.write([saveFrame], typeRegistry, null); + + expect(randomAccessBuffer.length, getFrameBytes([saveFrame])); + expect(randomAccessBuffer.writeOffset, getFrameBytes([saveFrame])); + + final storedValue = await randomAccessBuffer.read( + Frame("abc", "abc", offset: 0, length: 21), typeRegistry, null); + + expect(storedValue, "123"); + }); + + test('clear() resets random access buffer', () async { + final typeRegistry = TypeRegistryImpl.nullImpl; + + final randomAccessBuffer = RandomAccessBuffer(null); + randomAccessBuffer.recoveryCheck(typeRegistry, null, null); + + final frame = Frame("abc", "123"); + await randomAccessBuffer.write([frame], typeRegistry, null); + + expect(randomAccessBuffer.length, getFrameBytes([frame])); + expect(randomAccessBuffer.writeOffset, getFrameBytes([frame])); + + await randomAccessBuffer.clear(); + + expect(randomAccessBuffer.length, 0); + expect(randomAccessBuffer.writeOffset, 0); + }); + }); +} + +int getFrameBytes(Iterable frames) { + var writer = BinaryWriterImpl(TypeRegistryImpl.nullImpl); + for (var frame in frames) { + writer.writeFrame(frame); + } + return writer.toBytes().lengthInBytes; +} diff --git a/hive/test/tests/backend/memory/storage_backend_memory_test.dart b/hive/test/tests/backend/memory/storage_backend_memory_test.dart new file mode 100644 index 000000000..141fbe45e --- /dev/null +++ b/hive/test/tests/backend/memory/storage_backend_memory_test.dart @@ -0,0 +1,111 @@ +import 'dart:typed_data'; + +import 'package:hive/src/backend/stub/storage_backend_memory.dart'; +import 'package:hive/src/binary/binary_writer_impl.dart'; +import 'package:hive/src/binary/frame.dart'; +import 'package:hive/src/registry/type_registry_impl.dart'; +import 'package:test/test.dart'; + +import '../../common.dart'; +import '../../mocks.dart'; + +void main() { + group('StorageBackendMemory', () { + test('.path is null', () { + var backend = StorageBackendMemory(null, null); + expect(backend.path, null); + }); + + test('.supportsCompaction is false', () { + var backend = StorageBackendMemory(null, null); + expect(backend.supportsCompaction, false); + }); + + group('.initialize()', () { + test('throws if frames cannot be decoded', () { + var bytes = Uint8List.fromList([1, 2, 3, 4]); + var backend = StorageBackendMemory(bytes, null); + expect( + () => backend.initialize( + TypeRegistryImpl.nullImpl, KeystoreFake(), false), + throwsHiveError('Wrong checksum'), + ); + }); + }); + + test('.readValue() returns previously stored value', () async { + var backend = StorageBackendMemory(null, null); + await backend.initialize( + TypeRegistryImpl.nullImpl, MockKeystore(), false); + var testFrame = Frame('key', 'val'); + var frameBytes = getFrameBytes([testFrame]); + testFrame.length = frameBytes.lengthInBytes; + testFrame.offset = 0; + await backend.writeFrames([testFrame]); + expect( + await backend.readValue( + Frame('key', 'key', offset: 0, length: frameBytes.lengthInBytes)), + 'val'); + }); + + test('.readValue() returns previously stored value - multiple frames', + () async { + var backend = StorageBackendMemory(null, null); + await backend.initialize( + TypeRegistryImpl.nullImpl, MockKeystore(), false); + + var frame1 = Frame('key', 'val'); + var frame1Bytes = getFrameBytes([frame1]); + + var frame2 = Frame('123', 'abc'); + var frame2Bytes = getFrameBytes([frame2]); + + var frame3 = Frame('apfelkuchen', 'lecker'); + + await backend.writeFrames([frame1, frame2, frame3]); + final value = await backend.readValue(Frame('123', '123', + offset: frame1Bytes.length, length: frame2Bytes.lengthInBytes)); + expect('abc', value); + }); + + test('.writeFrames() writes data', () async { + var backend = StorageBackendMemory(null, null); + await backend.initialize( + TypeRegistryImpl.nullImpl, MockKeystore(), false); + await backend.writeFrames([Frame('key', 'val')]); + }); + + test('.compact() throws UnsupportedError', () { + var backend = StorageBackendMemory(null, null); + expect(() => backend.compact([]), throwsUnsupportedError); + }); + + test('.deleteFromDisk() does not throw', () async { + var backend = StorageBackendMemory(null, null); + await backend.deleteFromDisk(); + }); + + test('.clear() does not throw', () async { + var backend = StorageBackendMemory(null, null); + await backend.clear(); + }); + + test('.close() does not throw', () async { + var backend = StorageBackendMemory(null, null); + await backend.close(); + }); + + test('.flush() does nothing', () async { + var backend = StorageBackendMemory(null, null); + await backend.flush(); + }); + }); +} + +Uint8List getFrameBytes(Iterable frames) { + var writer = BinaryWriterImpl(TypeRegistryImpl.nullImpl); + for (var frame in frames) { + writer.writeFrame(frame); + } + return writer.toBytes(); +} diff --git a/hive/test/tests/backend/storage_backend_memory_test.dart b/hive/test/tests/backend/storage_backend_memory_test.dart deleted file mode 100644 index db89cb88b..000000000 --- a/hive/test/tests/backend/storage_backend_memory_test.dart +++ /dev/null @@ -1,64 +0,0 @@ -import 'dart:typed_data'; - -import 'package:hive/src/backend/storage_backend_memory.dart'; -import 'package:hive/src/binary/frame.dart'; -import 'package:hive/src/registry/type_registry_impl.dart'; -import 'package:test/test.dart'; - -import '../common.dart'; - -void main() { - group('StorageBackendMemory', () { - test('.path is null', () { - var backend = StorageBackendMemory(null, null); - expect(backend.path, null); - }); - - test('.supportsCompaction is false', () { - var backend = StorageBackendMemory(null, null); - expect(backend.supportsCompaction, false); - }); - - group('.initialize()', () { - test('throws if frames cannot be decoded', () { - var bytes = Uint8List.fromList([1, 2, 3, 4]); - var backend = StorageBackendMemory(bytes, null); - expect( - () => backend.initialize(TypeRegistryImpl.nullImpl, null, false), - throwsHiveError('Wrong checksum'), - ); - }); - }); - - test('.readValue() throws UnsupportedError', () { - var backend = StorageBackendMemory(null, null); - expect( - () => backend.readValue(Frame('key', 'val')), throwsUnsupportedError); - }); - - test('.writeFrames() does nothing', () async { - var backend = StorageBackendMemory(null, null); - await backend.writeFrames([Frame('key', 'val')]); - }); - - test('.compact() throws UnsupportedError', () { - var backend = StorageBackendMemory(null, null); - expect(() => backend.compact([]), throwsUnsupportedError); - }); - - test('.clear() does nothing', () async { - var backend = StorageBackendMemory(null, null); - await backend.clear(); - }); - - test('.close() does nothing', () async { - var backend = StorageBackendMemory(null, null); - await backend.close(); - }); - - test('.deleteFromDisk() throws UnsupportedError', () { - var backend = StorageBackendMemory(null, null); - expect(() => backend.deleteFromDisk(), throwsUnsupportedError); - }); - }); -}