From 6ee110e5ba49ea49d0f3d49b66b672f1d1d1ff98 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Sat, 4 May 2024 12:01:34 +0200 Subject: [PATCH] remove sled, upgrade rocksdb and rusqlite to conduit's version --- Cargo.lock | 286 ++++++++++++------------------- README.md | 27 +-- tools/iface/Cargo.toml | 14 +- tools/iface/src/db.rs | 2 - tools/iface/src/db/rocksdb.rs | 5 +- tools/iface/src/db/sled.rs | 73 -------- tools/migrate/Cargo.toml | 3 +- tools/migrate/src/main.rs | 12 +- tools/sled_to_sqlite/Cargo.toml | 11 -- tools/sled_to_sqlite/src/main.rs | 57 ------ 10 files changed, 127 insertions(+), 363 deletions(-) delete mode 100644 tools/iface/src/db/sled.rs delete mode 100644 tools/sled_to_sqlite/Cargo.toml delete mode 100644 tools/sled_to_sqlite/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index bc63912..3bd15ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,13 +4,14 @@ version = 3 [[package]] name = "ahash" -version = "0.7.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "getrandom", + "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -22,6 +23,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "ansi_term" version = "0.12.1" @@ -65,21 +72,22 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.59.2" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags", + "bitflags 2.5.0", "cexpr", "clang-sys", + "itertools", "lazy_static", "lazycell", - "peeking_take_while", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", + "syn", ] [[package]] @@ -88,6 +96,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "bytemuck" version = "1.14.3" @@ -100,6 +114,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.88" @@ -143,7 +168,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", - "bitflags", + "bitflags 1.3.2", "strsim", "textwrap", "unicode-width", @@ -158,9 +183,8 @@ dependencies = [ "heed", "itertools", "persy", - "rocksdb", "rusqlite", - "sled", + "rust-rocksdb", "thiserror", ] @@ -174,15 +198,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "conduit_sled_to_sqlite" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "conduit_iface", -] - [[package]] name = "crc" version = "3.0.1" @@ -198,24 +213,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" -[[package]] -name = "crc32fast" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -245,9 +242,9 @@ checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "fallible-iterator" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fallible-streaming-iterator" @@ -274,15 +271,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "getrandom" version = "0.2.12" @@ -302,18 +290,19 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", + "allocator-api2", ] [[package]] name = "hashlink" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" dependencies = [ "hashbrown", ] @@ -373,15 +362,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "itertools" version = "0.10.5" @@ -426,22 +406,21 @@ dependencies = [ ] [[package]] -name = "librocksdb-sys" -version = "6.20.3" +name = "libsqlite3-sys" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" dependencies = [ - "bindgen", "cc", - "glob", - "libc", + "pkg-config", + "vcpkg", ] [[package]] -name = "libsqlite3-sys" -version = "0.22.2" +name = "libz-sys" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" dependencies = [ "cc", "pkg-config", @@ -466,21 +445,15 @@ dependencies = [ ] [[package]] -name = "lock_api" -version = "0.4.11" +name = "lz4-sys" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" dependencies = [ - "autocfg", - "scopeguard", + "cc", + "libc", ] -[[package]] -name = "log" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" - [[package]] name = "memchr" version = "2.7.1" @@ -528,37 +501,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", -] - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -641,15 +583,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.10.3" @@ -679,31 +612,46 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" -[[package]] -name = "rocksdb" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7" -dependencies = [ - "libc", - "librocksdb-sys", -] - [[package]] name = "rusqlite" -version = "0.25.4" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4b1eaf239b47034fb450ee9cdedd7d0226571689d8823030c4b6c2cb407152" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" dependencies = [ - "bitflags", + "bitflags 2.5.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "memchr", "smallvec", ] +[[package]] +name = "rust-librocksdb-sys" +version = "0.21.0+9.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cb7b9cd5ce3b3ce0757ceab2240f7471826780b8700845c0cfd418cb7e398d" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "rust-rocksdb" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bcfb31b5bf2e3274686ebfdf9a946e9a327a3bc54adc7e5cda9f4fdcc4b55f1" +dependencies = [ + "libc", + "rust-librocksdb-sys", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -716,12 +664,6 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "serde" version = "1.0.197" @@ -759,23 +701,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot", - "zstd", -] - [[package]] name = "smallvec" version = "1.13.1" @@ -1008,6 +933,26 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zigzag" version = "0.1.0" @@ -1017,31 +962,12 @@ dependencies = [ "num-traits", ] -[[package]] -name = "zstd" -version = "0.9.2+zstd.1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "4.1.3+zstd.1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79" -dependencies = [ - "libc", - "zstd-sys", -] - [[package]] name = "zstd-sys" -version = "1.6.2+zstd.1.5.1" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", - "libc", + "pkg-config", ] diff --git a/README.md b/README.md index bf6da6c..19e3a59 100644 --- a/README.md +++ b/README.md @@ -8,27 +8,11 @@ This repository provides binaries to wrangle your conduit homeserver with. ### `conduit_sled_to_sqlite` -A one-shot migration tool to convert your sled database to a sqlite one; +**This tool is no longer available on the main branch**, go to `with-sled` to download from there. -Instructions: (read the notes below first) -1. execute `conduit_sled_to_sqlite ` - - `conduit_sled_to_sqlite /var/lib/matrix-conduit/conduit_db` -2. the console will be spammed with a bunch of numbers, this is for diagnostic purposes, when the command returns, your database has been converted. +### `conduit_migrate` -Some notes: -- The conversion process can take a second depending on the speed of the disk of your server, on raspberry pis it could take a while. -- **Have your server turned off** throughout the conversion process, be sure it is updated to a version which uses SQLite as the main database, and only then turn it back on - - If you have the server be on during the conversion process, some data might be missing from the new database - - If you turn the server back on with an older version (which still uses sled), it'll create a brand new sled database in the database directory, which might be confusing. -- The above method places the new database in the same directory as the old one, this'll effectively duplicate your database, if you're sure your server is up-to-date and works with the new database, you can remove the old sled database files under that directory (be sure you have a backup first!); - - `blobs/` - - `db` - - `conf` -- The new database can be anywhere from 2 to 4 times bigger than the old database, if you want to know how much your current database size is, run the following command; - - `du --apparent-size --max-depth=0 -h ` - - And if you want to know how much room you have on your disk, you can run this; - - `df -h /` (with that trailing slash) - - Make sure you have this disk-space *extra* on your machine, else the conversion process will fill it up. +This tool provides generic migration between `heed`, `sqlite`, `persy`, and `rocksdb` conduit databases. ## Installing @@ -38,9 +22,6 @@ For the best experience, compile this toolbox locally on your server; 2. Be sure that the rust executables are on your `$PATH` 3. You may want to have a compiler and build tools installed on your system, or else cargo will complain about not being able to "link" or "compile" with `cc`. - on debian/ubuntu-based systems you can install this with `sudo apt install build-essential` -5. `cargo install --locked --git https://github.com/shadowjonathan/conduit_toolbox` (Plus an extra argument, see below) - - Depending on what you're trying to do, you'll have to add one more argument here - - Are you trying to install the Sled-to-Sqlite migrator? Add `conduit_sled_to_sqlite` - - Are you trying to install the generic migrator? Add `conduit_migrate` +4. `cargo install --locked --git https://github.com/shadowjonathan/conduit_toolbox conduit_migrate` (updating only requires running that last line again) diff --git a/tools/iface/Cargo.toml b/tools/iface/Cargo.toml index 605ccde..15c3b7b 100644 --- a/tools/iface/Cargo.toml +++ b/tools/iface/Cargo.toml @@ -10,12 +10,20 @@ itertools = "0.10.1" thiserror = "1.0.26" anyhow = "1.0.42" -sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true } -rusqlite = { version = "0.25.3", features = ["bundled"], optional = true } +rusqlite = { version = "0.31", features = ["bundled"], optional = true } heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", features = ["multi-threaded-cf", "zstd"], optional = true } persy = { version = "1.2", optional = true } +[dependencies.rocksdb] +package = "rust-rocksdb" +version = "0.25" +optional = true +features = [ + "multi-threaded-cf", + "zstd", + "lz4", +] + [features] default = [] diff --git a/tools/iface/src/db.rs b/tools/iface/src/db.rs index f87dc9c..2a9d310 100644 --- a/tools/iface/src/db.rs +++ b/tools/iface/src/db.rs @@ -4,8 +4,6 @@ pub mod heed; pub mod persy; #[cfg(feature = "rocksdb")] pub mod rocksdb; -#[cfg(feature = "sled")] -pub mod sled; #[cfg(feature = "sqlite")] pub mod sqlite; diff --git a/tools/iface/src/db/rocksdb.rs b/tools/iface/src/db/rocksdb.rs index 63a3594..f3fdf40 100644 --- a/tools/iface/src/db/rocksdb.rs +++ b/tools/iface/src/db/rocksdb.rs @@ -131,7 +131,10 @@ impl super::SegmentIter for RocksDBCFIter<'_> { .db .rocks .iterator_cf(&self.0.cf(), rocksdb::IteratorMode::Start) - .map(|(k, v)| (Vec::from(k), Vec::from(v))), + .map(|r| { + let (k, v) = r.expect("we expect rocksdb to give us good rows only"); + (Vec::from(k), Vec::from(v)) + }), ) } } diff --git a/tools/iface/src/db/sled.rs b/tools/iface/src/db/sled.rs deleted file mode 100644 index f2d465f..0000000 --- a/tools/iface/src/db/sled.rs +++ /dev/null @@ -1,73 +0,0 @@ -use super::{Database, KVIter, Segment, SegmentIter}; -use itertools::Itertools; -use sled::{Batch, Config, Db, Result, Tree}; -use std::path::Path; - -pub fn new_db>(path: P) -> Result { - Config::default().path(path).use_compression(true).open() -} - -pub struct SledDB(Db); - -impl SledDB { - pub fn new(db: Db) -> Self { - Self(db) - } -} - -const SLED_DEFAULT: &[u8] = "__sled__default".as_bytes(); - -impl Database for SledDB { - fn names<'a>(&'a self) -> Vec> { - self.0 - .tree_names() - .into_iter() - .filter(|v| v != SLED_DEFAULT) - .map(|v| v.to_vec()) - .collect_vec() - } - - fn segment(&mut self, name: Vec) -> Option> { - self.0 - .open_tree(name) - .ok() - .map(|t| -> Box { Box::new(t) }) - } - - fn flush(&mut self) { - self.0.flush().unwrap(); - } -} - -impl Segment for Tree { - fn batch_insert( - &mut self, - batch: Box, Vec)> + '_>, - ) -> anyhow::Result<()> { - let mut sled_batch = Batch::default(); - - for (k, v) in batch { - sled_batch.insert(k, v) - } - - self.apply_batch(sled_batch).map_err(Into::into) - } - - fn get_iter<'a>(&'a mut self) -> Box { - Box::new(SledTreeIter(self)) - } -} - -struct SledTreeIter<'a>(&'a mut Tree); - -impl SegmentIter for SledTreeIter<'_> { - fn iter<'a>(&'a mut self) -> KVIter<'a> { - Box::new(self.0.iter().filter_map(|r| { - if let Ok(t) = r { - Some((t.0.to_vec(), t.1.to_vec())) - } else { - None - } - })) - } -} diff --git a/tools/migrate/Cargo.toml b/tools/migrate/Cargo.toml index cfcd743..2082560 100644 --- a/tools/migrate/Cargo.toml +++ b/tools/migrate/Cargo.toml @@ -12,9 +12,8 @@ conduit_iface = { path = "../iface/", default-features = false } thiserror = "1.0.26" [features] -default = ["sled", "sqlite", "rocksdb"] +default = ["sqlite", "rocksdb"] -sled = ["conduit_iface/sled"] persy = ["conduit_iface/persy"] heed = ["conduit_iface/heed"] sqlite = ["conduit_iface/sqlite"] diff --git a/tools/migrate/src/main.rs b/tools/migrate/src/main.rs index 0ba5593..f05cfb0 100644 --- a/tools/migrate/src/main.rs +++ b/tools/migrate/src/main.rs @@ -6,8 +6,6 @@ use std::{ }; enum Database { - #[cfg(feature = "sled")] - Sled(db::sled::SledDB), #[cfg(feature = "heed")] Heed(db::heed::HeedDB), #[cfg(feature = "sqlite")] @@ -21,8 +19,6 @@ enum Database { impl Database { fn new(name: &str, path: PathBuf, config: Config) -> anyhow::Result { Ok(match name { - #[cfg(feature = "sled")] - "sled" => Self::Sled(db::sled::SledDB::new(db::sled::new_db(path)?)), #[cfg(feature = "heed")] "heed" => Self::Heed(db::heed::HeedDB::new(db::heed::new_db(path)?)), #[cfg(feature = "sqlite")] @@ -44,8 +40,6 @@ impl Deref for Database { fn deref(&self) -> &Self::Target { match self { - #[cfg(feature = "sled")] - Database::Sled(db) => db, #[cfg(feature = "heed")] Database::Heed(db) => db, #[cfg(feature = "sqlite")] @@ -61,8 +55,6 @@ impl Deref for Database { impl DerefMut for Database { fn deref_mut(&mut self) -> &mut Self::Target { match self { - #[cfg(feature = "sled")] - Database::Sled(db) => db, #[cfg(feature = "heed")] Database::Heed(db) => db, #[cfg(feature = "sqlite")] @@ -76,8 +68,6 @@ impl DerefMut for Database { } const DATABASES: &[&str] = &[ - #[cfg(feature = "sled")] - "sled", #[cfg(feature = "heed")] "heed", #[cfg(feature = "sqlite")] @@ -89,7 +79,7 @@ const DATABASES: &[&str] = &[ ]; fn main() -> anyhow::Result<()> { - let matches = App::new("Conduit Sled to Sqlite Migrator") + let matches = App::new("Conduit Generic Migrator") .arg( Arg::with_name("from_dir") .short("s") diff --git a/tools/sled_to_sqlite/Cargo.toml b/tools/sled_to_sqlite/Cargo.toml deleted file mode 100644 index c0e3fd8..0000000 --- a/tools/sled_to_sqlite/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "conduit_sled_to_sqlite" -version = "0.1.0" -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -clap = "2.33.3" -anyhow = "1.0.41" -conduit_iface = { path = "../iface/", features=["sled", "sqlite"] } diff --git a/tools/sled_to_sqlite/src/main.rs b/tools/sled_to_sqlite/src/main.rs deleted file mode 100644 index 77a74f0..0000000 --- a/tools/sled_to_sqlite/src/main.rs +++ /dev/null @@ -1,57 +0,0 @@ -use std::path::Path; - -use clap::{App, Arg}; - -use conduit_iface::db::{copy_database, sled, sqlite, Config}; - -fn main() -> anyhow::Result<()> { - let matches = App::new("Conduit Sled to Sqlite Migrator") - .arg( - Arg::with_name("DIR") - .long_help("Sets the directory to grab the database from\nWill default to \".\"") - .index(1), - ) - .arg( - Arg::with_name("NEW_DIR") - .long_help("Sets the destination directory\nWill default to DIR") - .index(2), - ) - .get_matches(); - - let source_dir = matches.value_of("DIR").unwrap_or("."); - - let dest_dir = matches.value_of("NEW_DIR"); - - let source_dir = Path::new(source_dir).canonicalize()?; - - if !source_dir.is_dir() { - return Err(anyhow::anyhow!("source path must be directory")); - } - - let dest_dir = match dest_dir { - None => Ok(source_dir.clone()), - Some(dir) => { - let p = Path::new(dir).canonicalize()?; - if !p.is_dir() { - Err(anyhow::anyhow!("destination path must be directory")) - } else { - Ok(p) - } - } - }?; - - dbg!(&source_dir, &dest_dir); - - let mut sled = sled::SledDB::new(sled::new_db(source_dir)?); - - let mut sqlite = sqlite::SqliteDB::new( - sqlite::new_conn(dest_dir)?, - Config { - ignore_broken_rows: false, - }, - ); - - copy_database(&mut sled, &mut sqlite, 1000)?; - - Ok(()) -}