mirror of
https://github.com/ShadowJonathan/conduit_toolbox.git
synced 2025-06-07 09:57:02 +03:00
remove sled, upgrade rocksdb and rusqlite to conduit's version
This commit is contained in:
Jonathan de Jong
parent
7cc4616fbf
commit
6ee110e5ba
10 changed files with 127 additions and 363 deletions
286
Cargo.lock
generated
286
Cargo.lock
generated
|
|
@ -4,13 +4,14 @@ version = 3
|
|||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.7.8"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
|
||||
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -22,6 +23,12 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
|
||||
|
||||
[[package]]
|
||||
name = "ansi_term"
|
||||
version = "0.12.1"
|
||||
|
|
@ -65,21 +72,22 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.59.2"
|
||||
version = "0.69.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8"
|
||||
checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bitflags 2.5.0",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"itertools",
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"peeking_take_while",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -88,6 +96,12 @@ version = "1.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.14.3"
|
||||
|
|
@ -100,6 +114,17 @@ version = "1.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bzip2-sys"
|
||||
version = "0.1.11+1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.88"
|
||||
|
|
@ -143,7 +168,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
|
|||
dependencies = [
|
||||
"ansi_term",
|
||||
"atty",
|
||||
"bitflags",
|
||||
"bitflags 1.3.2",
|
||||
"strsim",
|
||||
"textwrap",
|
||||
"unicode-width",
|
||||
|
|
@ -158,9 +183,8 @@ dependencies = [
|
|||
"heed",
|
||||
"itertools",
|
||||
"persy",
|
||||
"rocksdb",
|
||||
"rusqlite",
|
||||
"sled",
|
||||
"rust-rocksdb",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
|
|
@ -174,15 +198,6 @@ dependencies = [
|
|||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "conduit_sled_to_sqlite"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"conduit_iface",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc"
|
||||
version = "3.0.1"
|
||||
|
|
@ -198,24 +213,6 @@ version = "2.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.11"
|
||||
|
|
@ -245,9 +242,9 @@ checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
|
|||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.2.0"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
|
|
@ -274,15 +271,6 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fxhash"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.12"
|
||||
|
|
@ -302,18 +290,19 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
|
|||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.11.2"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"allocator-api2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.7.0"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
|
||||
checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee"
|
||||
dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
|
@ -373,15 +362,6 @@ dependencies = [
|
|||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
|
|
@ -426,22 +406,21 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "librocksdb-sys"
|
||||
version = "6.20.3"
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d"
|
||||
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
"glob",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.22.2"
|
||||
name = "libz-sys"
|
||||
version = "1.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d"
|
||||
checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
|
|
@ -466,21 +445,15 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.11"
|
||||
name = "lz4-sys"
|
||||
version = "1.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
|
||||
checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.1"
|
||||
|
|
@ -528,37 +501,6 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.1"
|
||||
|
|
@ -641,15 +583,6 @@ dependencies = [
|
|||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.10.3"
|
||||
|
|
@ -679,31 +612,46 @@ version = "0.8.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
|
||||
|
||||
[[package]]
|
||||
name = "rocksdb"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"librocksdb-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.25.4"
|
||||
version = "0.31.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c4b1eaf239b47034fb450ee9cdedd7d0226571689d8823030c4b6c2cb407152"
|
||||
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bitflags 2.5.0",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"memchr",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-librocksdb-sys"
|
||||
version = "0.21.0+9.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75cb7b9cd5ce3b3ce0757ceab2240f7471826780b8700845c0cfd418cb7e398d"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"bzip2-sys",
|
||||
"cc",
|
||||
"glob",
|
||||
"libc",
|
||||
"libz-sys",
|
||||
"lz4-sys",
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-rocksdb"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bcfb31b5bf2e3274686ebfdf9a946e9a327a3bc54adc7e5cda9f4fdcc4b55f1"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rust-librocksdb-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
|
|
@ -716,12 +664,6 @@ version = "1.0.17"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.197"
|
||||
|
|
@ -759,23 +701,6 @@ version = "1.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "sled"
|
||||
version = "0.34.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"fs2",
|
||||
"fxhash",
|
||||
"libc",
|
||||
"log",
|
||||
"parking_lot",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.13.1"
|
||||
|
|
@ -1008,6 +933,26 @@ version = "0.48.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zigzag"
|
||||
version = "0.1.0"
|
||||
|
|
@ -1017,31 +962,12 @@ dependencies = [
|
|||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.9.2+zstd.1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "4.1.3+zstd.1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "1.6.2+zstd.1.5.1"
|
||||
version = "2.0.10+zstd.1.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f"
|
||||
checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
|
|
|||
27
README.md
27
README.md
|
|
@ -8,27 +8,11 @@ This repository provides binaries to wrangle your conduit homeserver with.
|
|||
|
||||
### `conduit_sled_to_sqlite`
|
||||
|
||||
A one-shot migration tool to convert your sled database to a sqlite one;
|
||||
**This tool is no longer available on the main branch**, go to `with-sled` to download from there.
|
||||
|
||||
Instructions: (read the notes below first)
|
||||
1. execute `conduit_sled_to_sqlite <database_directory>`
|
||||
- `conduit_sled_to_sqlite /var/lib/matrix-conduit/conduit_db`
|
||||
2. the console will be spammed with a bunch of numbers, this is for diagnostic purposes, when the command returns, your database has been converted.
|
||||
### `conduit_migrate`
|
||||
|
||||
Some notes:
|
||||
- The conversion process can take a second depending on the speed of the disk of your server, on raspberry pis it could take a while.
|
||||
- **Have your server turned off** throughout the conversion process, be sure it is updated to a version which uses SQLite as the main database, and only then turn it back on
|
||||
- If you have the server be on during the conversion process, some data might be missing from the new database
|
||||
- If you turn the server back on with an older version (which still uses sled), it'll create a brand new sled database in the database directory, which might be confusing.
|
||||
- The above method places the new database in the same directory as the old one, this'll effectively duplicate your database, if you're sure your server is up-to-date and works with the new database, you can remove the old sled database files under that directory (be sure you have a backup first!);
|
||||
- `blobs/`
|
||||
- `db`
|
||||
- `conf`
|
||||
- The new database can be anywhere from 2 to 4 times bigger than the old database, if you want to know how much your current database size is, run the following command;
|
||||
- `du --apparent-size --max-depth=0 -h <database_directory>`
|
||||
- And if you want to know how much room you have on your disk, you can run this;
|
||||
- `df -h <database_directory>/` (with that trailing slash)
|
||||
- Make sure you have this disk-space *extra* on your machine, else the conversion process will fill it up.
|
||||
This tool provides generic migration between `heed`, `sqlite`, `persy`, and `rocksdb` conduit databases.
|
||||
|
||||
## Installing
|
||||
|
||||
|
|
@ -38,9 +22,6 @@ For the best experience, compile this toolbox locally on your server;
|
|||
2. Be sure that the rust executables are on your `$PATH`
|
||||
3. You may want to have a compiler and build tools installed on your system, or else cargo will complain about not being able to "link" or "compile" with `cc`.
|
||||
- on debian/ubuntu-based systems you can install this with `sudo apt install build-essential`
|
||||
5. `cargo install --locked --git https://github.com/shadowjonathan/conduit_toolbox` (Plus an extra argument, see below)
|
||||
- Depending on what you're trying to do, you'll have to add one more argument here
|
||||
- Are you trying to install the Sled-to-Sqlite migrator? Add `conduit_sled_to_sqlite`
|
||||
- Are you trying to install the generic migrator? Add `conduit_migrate`
|
||||
4. `cargo install --locked --git https://github.com/shadowjonathan/conduit_toolbox conduit_migrate`
|
||||
|
||||
(updating only requires running that last line again)
|
||||
|
|
|
|||
|
|
@ -10,12 +10,20 @@ itertools = "0.10.1"
|
|||
thiserror = "1.0.26"
|
||||
anyhow = "1.0.42"
|
||||
|
||||
sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true }
|
||||
rusqlite = { version = "0.25.3", features = ["bundled"], optional = true }
|
||||
rusqlite = { version = "0.31", features = ["bundled"], optional = true }
|
||||
heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
|
||||
rocksdb = { version = "0.17.0", features = ["multi-threaded-cf", "zstd"], optional = true }
|
||||
persy = { version = "1.2", optional = true }
|
||||
|
||||
[dependencies.rocksdb]
|
||||
package = "rust-rocksdb"
|
||||
version = "0.25"
|
||||
optional = true
|
||||
features = [
|
||||
"multi-threaded-cf",
|
||||
"zstd",
|
||||
"lz4",
|
||||
]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ pub mod heed;
|
|||
pub mod persy;
|
||||
#[cfg(feature = "rocksdb")]
|
||||
pub mod rocksdb;
|
||||
#[cfg(feature = "sled")]
|
||||
pub mod sled;
|
||||
#[cfg(feature = "sqlite")]
|
||||
pub mod sqlite;
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,10 @@ impl super::SegmentIter for RocksDBCFIter<'_> {
|
|||
.db
|
||||
.rocks
|
||||
.iterator_cf(&self.0.cf(), rocksdb::IteratorMode::Start)
|
||||
.map(|(k, v)| (Vec::from(k), Vec::from(v))),
|
||||
.map(|r| {
|
||||
let (k, v) = r.expect("we expect rocksdb to give us good rows only");
|
||||
(Vec::from(k), Vec::from(v))
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
use super::{Database, KVIter, Segment, SegmentIter};
|
||||
use itertools::Itertools;
|
||||
use sled::{Batch, Config, Db, Result, Tree};
|
||||
use std::path::Path;
|
||||
|
||||
pub fn new_db<P: AsRef<Path>>(path: P) -> Result<Db> {
|
||||
Config::default().path(path).use_compression(true).open()
|
||||
}
|
||||
|
||||
pub struct SledDB(Db);
|
||||
|
||||
impl SledDB {
|
||||
pub fn new(db: Db) -> Self {
|
||||
Self(db)
|
||||
}
|
||||
}
|
||||
|
||||
const SLED_DEFAULT: &[u8] = "__sled__default".as_bytes();
|
||||
|
||||
impl Database for SledDB {
|
||||
fn names<'a>(&'a self) -> Vec<Vec<u8>> {
|
||||
self.0
|
||||
.tree_names()
|
||||
.into_iter()
|
||||
.filter(|v| v != SLED_DEFAULT)
|
||||
.map(|v| v.to_vec())
|
||||
.collect_vec()
|
||||
}
|
||||
|
||||
fn segment(&mut self, name: Vec<u8>) -> Option<Box<dyn Segment>> {
|
||||
self.0
|
||||
.open_tree(name)
|
||||
.ok()
|
||||
.map(|t| -> Box<dyn Segment> { Box::new(t) })
|
||||
}
|
||||
|
||||
fn flush(&mut self) {
|
||||
self.0.flush().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
impl Segment for Tree {
|
||||
fn batch_insert(
|
||||
&mut self,
|
||||
batch: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + '_>,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut sled_batch = Batch::default();
|
||||
|
||||
for (k, v) in batch {
|
||||
sled_batch.insert(k, v)
|
||||
}
|
||||
|
||||
self.apply_batch(sled_batch).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn get_iter<'a>(&'a mut self) -> Box<dyn super::SegmentIter + 'a> {
|
||||
Box::new(SledTreeIter(self))
|
||||
}
|
||||
}
|
||||
|
||||
struct SledTreeIter<'a>(&'a mut Tree);
|
||||
|
||||
impl SegmentIter for SledTreeIter<'_> {
|
||||
fn iter<'a>(&'a mut self) -> KVIter<'a> {
|
||||
Box::new(self.0.iter().filter_map(|r| {
|
||||
if let Ok(t) = r {
|
||||
Some((t.0.to_vec(), t.1.to_vec()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
|
@ -12,9 +12,8 @@ conduit_iface = { path = "../iface/", default-features = false }
|
|||
thiserror = "1.0.26"
|
||||
|
||||
[features]
|
||||
default = ["sled", "sqlite", "rocksdb"]
|
||||
default = ["sqlite", "rocksdb"]
|
||||
|
||||
sled = ["conduit_iface/sled"]
|
||||
persy = ["conduit_iface/persy"]
|
||||
heed = ["conduit_iface/heed"]
|
||||
sqlite = ["conduit_iface/sqlite"]
|
||||
|
|
|
|||
|
|
@ -6,8 +6,6 @@ use std::{
|
|||
};
|
||||
|
||||
enum Database {
|
||||
#[cfg(feature = "sled")]
|
||||
Sled(db::sled::SledDB),
|
||||
#[cfg(feature = "heed")]
|
||||
Heed(db::heed::HeedDB),
|
||||
#[cfg(feature = "sqlite")]
|
||||
|
|
@ -21,8 +19,6 @@ enum Database {
|
|||
impl Database {
|
||||
fn new(name: &str, path: PathBuf, config: Config) -> anyhow::Result<Self> {
|
||||
Ok(match name {
|
||||
#[cfg(feature = "sled")]
|
||||
"sled" => Self::Sled(db::sled::SledDB::new(db::sled::new_db(path)?)),
|
||||
#[cfg(feature = "heed")]
|
||||
"heed" => Self::Heed(db::heed::HeedDB::new(db::heed::new_db(path)?)),
|
||||
#[cfg(feature = "sqlite")]
|
||||
|
|
@ -44,8 +40,6 @@ impl Deref for Database {
|
|||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
match self {
|
||||
#[cfg(feature = "sled")]
|
||||
Database::Sled(db) => db,
|
||||
#[cfg(feature = "heed")]
|
||||
Database::Heed(db) => db,
|
||||
#[cfg(feature = "sqlite")]
|
||||
|
|
@ -61,8 +55,6 @@ impl Deref for Database {
|
|||
impl DerefMut for Database {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
match self {
|
||||
#[cfg(feature = "sled")]
|
||||
Database::Sled(db) => db,
|
||||
#[cfg(feature = "heed")]
|
||||
Database::Heed(db) => db,
|
||||
#[cfg(feature = "sqlite")]
|
||||
|
|
@ -76,8 +68,6 @@ impl DerefMut for Database {
|
|||
}
|
||||
|
||||
const DATABASES: &[&str] = &[
|
||||
#[cfg(feature = "sled")]
|
||||
"sled",
|
||||
#[cfg(feature = "heed")]
|
||||
"heed",
|
||||
#[cfg(feature = "sqlite")]
|
||||
|
|
@ -89,7 +79,7 @@ const DATABASES: &[&str] = &[
|
|||
];
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let matches = App::new("Conduit Sled to Sqlite Migrator")
|
||||
let matches = App::new("Conduit Generic Migrator")
|
||||
.arg(
|
||||
Arg::with_name("from_dir")
|
||||
.short("s")
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
[package]
|
||||
name = "conduit_sled_to_sqlite"
|
||||
version = "0.1.0"
|
||||
edition = "2018"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
clap = "2.33.3"
|
||||
anyhow = "1.0.41"
|
||||
conduit_iface = { path = "../iface/", features=["sled", "sqlite"] }
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
use std::path::Path;
|
||||
|
||||
use clap::{App, Arg};
|
||||
|
||||
use conduit_iface::db::{copy_database, sled, sqlite, Config};
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let matches = App::new("Conduit Sled to Sqlite Migrator")
|
||||
.arg(
|
||||
Arg::with_name("DIR")
|
||||
.long_help("Sets the directory to grab the database from\nWill default to \".\"")
|
||||
.index(1),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("NEW_DIR")
|
||||
.long_help("Sets the destination directory\nWill default to DIR")
|
||||
.index(2),
|
||||
)
|
||||
.get_matches();
|
||||
|
||||
let source_dir = matches.value_of("DIR").unwrap_or(".");
|
||||
|
||||
let dest_dir = matches.value_of("NEW_DIR");
|
||||
|
||||
let source_dir = Path::new(source_dir).canonicalize()?;
|
||||
|
||||
if !source_dir.is_dir() {
|
||||
return Err(anyhow::anyhow!("source path must be directory"));
|
||||
}
|
||||
|
||||
let dest_dir = match dest_dir {
|
||||
None => Ok(source_dir.clone()),
|
||||
Some(dir) => {
|
||||
let p = Path::new(dir).canonicalize()?;
|
||||
if !p.is_dir() {
|
||||
Err(anyhow::anyhow!("destination path must be directory"))
|
||||
} else {
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
}?;
|
||||
|
||||
dbg!(&source_dir, &dest_dir);
|
||||
|
||||
let mut sled = sled::SledDB::new(sled::new_db(source_dir)?);
|
||||
|
||||
let mut sqlite = sqlite::SqliteDB::new(
|
||||
sqlite::new_conn(dest_dir)?,
|
||||
Config {
|
||||
ignore_broken_rows: false,
|
||||
},
|
||||
);
|
||||
|
||||
copy_database(&mut sled, &mut sqlite, 1000)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue