# Copyright (c) 2016-2017, the ElectrumX authors # # All rights reserved. # # See the file "LICENCE" for information about the copyright # and warranty status of this software. '''Backend database abstraction.''' import os from functools import partial from typing import Type, Sequence, TYPE_CHECKING import electrumx.lib.util as util if TYPE_CHECKING: from rocksdb import DB as rocksdb_db from plyvel import DB as plyvel_db def db_class(name: str) -> Type['Storage']: '''Returns a DB engine class.''' for db_class in util.subclasses(Storage): if db_class.__name__.lower() == name.lower(): db_class.import_module() return db_class raise RuntimeError(f'unrecognised DB engine "{name}"') def list_db_engine_choices() -> Sequence[str]: return [db_class.__name__.lower() for db_class in util.subclasses(Storage)] class Storage: '''Abstract base class of the DB backend abstraction.''' def __init__(self, name: str, for_sync: bool): self.is_new = not os.path.exists(name) self.for_sync = for_sync or self.is_new self.open(name, create=self.is_new) @classmethod def import_module(cls) -> None: '''Import the DB engine module.''' raise NotImplementedError def open(self, name: str, create: bool): '''Open an existing database or create a new one.''' raise NotImplementedError def close(self) -> None: '''Close an existing database.''' raise NotImplementedError def get(self, key: bytes) -> bytes: raise NotImplementedError def put(self, key: bytes, value: bytes) -> None: raise NotImplementedError def write_batch(self): '''Return a context manager that provides `put` and `delete`. Changes should only be committed when the context manager closes without an exception. ''' raise NotImplementedError def iterator(self, prefix: bytes = b'', reverse: bool = False): '''Return an iterator that yields (key, value) pairs from the database sorted by key. If `prefix` is set, only keys starting with `prefix` will be included. If `reverse` is True the items are returned in reverse order. The iterator supports .seek(key), which moves it just left of `key`. - if forward-iterating - if `key` is present, it will be the next item - if `key` is not present, the next item will be the smallest still greater than `key` - if reverse-iterating - the next item will be the largest still smaller than `key` ''' raise NotImplementedError class LevelDB(Storage): '''LevelDB database engine.''' db: 'plyvel_db' @classmethod def import_module(cls): try: import plyvel except ImportError as e: raise Exception(f"{e}. For the leveldb DB-backend, you should install ElectrumX with the [leveldb] pip extra.") from e cls.module = plyvel def open(self, name, create): mof = 512 if self.for_sync else 128 # Use snappy compression (the default) self.db = self.module.DB(name, create_if_missing=create, max_open_files=mof) self.close = self.db.close self.get = self.db.get self.put = self.db.put self.write_batch = partial(self.db.write_batch, transaction=True, sync=True) def iterator(self, prefix=b'', reverse=False): return LevelDBIterator(db=self.db, prefix=prefix, reverse=reverse) class LevelDBIterator: '''An iterator for LevelDB.''' def __init__(self, *, db: 'plyvel_db', prefix: bytes, reverse: bool): self.prefix = prefix self.iterator = db.iterator(prefix=prefix, reverse=reverse) def __iter__(self): return self def __next__(self): k, v = next(self.iterator) if not k.startswith(self.prefix): raise StopIteration return k, v def seek(self, key: bytes) -> None: self.iterator.seek(key) class RocksDB(Storage): '''RocksDB database engine.''' db: 'rocksdb_db' @classmethod def import_module(cls): try: import rocksdb except ImportError as e: raise Exception(f"{e}. For the rocksdb DB-backend, you should install ElectrumX with the [rocksdb] pip extra.") from e cls.module = rocksdb def open(self, name, create): mof = 512 if self.for_sync else 128 # Use snappy compression (the default) options = self.module.Options(create_if_missing=create, use_fsync=True, target_file_size_base=33554432, max_open_files=mof) self.db = self.module.DB(name, options) self.close = self.db.close self.get = self.db.get self.put = self.db.put def write_batch(self): return RocksDBWriteBatch(self.db) def iterator(self, prefix=b'', reverse=False): return RocksDBIterator(db=self.db, prefix=prefix, reverse=reverse) class RocksDBWriteBatch: '''A write batch for RocksDB.''' def __init__(self, db: 'rocksdb_db'): self.batch = RocksDB.module.WriteBatch() self.db = db def __enter__(self): return self.batch def __exit__(self, exc_type, exc_val, exc_tb): if not exc_val: self.db.write(self.batch) class RocksDBIterator: '''An iterator for RocksDB.''' def __init__(self, *, db: 'rocksdb_db', prefix: bytes, reverse: bool): self.prefix = prefix self._is_reverse = reverse if reverse: self.iterator = reversed(db.iteritems()) nxt_prefix = util.increment_byte_string(prefix) if nxt_prefix: self.iterator.seek(nxt_prefix) try: next(self.iterator) except StopIteration: self.iterator.seek(nxt_prefix) else: self.iterator.seek_to_last() else: self.iterator = db.iteritems() self.iterator.seek(prefix) def __iter__(self): return self def __next__(self): k, v = next(self.iterator) if not k.startswith(self.prefix): raise StopIteration return k, v def seek(self, key: bytes) -> None: self.iterator.seek(key) if self._is_reverse: try: next(self) except StopIteration: self.iterator.seek_to_last()