mirror of
https://github.com/ShadowJonathan/conduit_toolbox.git
synced 2025-07-10 12:04:38 +03:00
INIT
This commit is contained in:
commit
03fde79046
7 changed files with 671 additions and 0 deletions
13
tools/sled_to_sqlite/Cargo.toml
Normal file
13
tools/sled_to_sqlite/Cargo.toml
Normal file
|
@ -0,0 +1,13 @@
|
|||
[package]
|
||||
name = "conduit_sled_to_sqlite"
|
||||
version = "0.1.0"
|
||||
edition = "2018"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
sled = { version = "0.34.6", features = ["compression", "no_metrics"] }
|
||||
rusqlite = { version = "0.25.3", features = ["bundled"] }
|
||||
clap = "2.33.3"
|
||||
anyhow = "1.0.41"
|
||||
itertools = "0.10.1"
|
70
tools/sled_to_sqlite/src/main.rs
Normal file
70
tools/sled_to_sqlite/src/main.rs
Normal file
|
@ -0,0 +1,70 @@
|
|||
mod sled;
|
||||
mod sqlite;
|
||||
use std::path::Path;
|
||||
|
||||
use clap::{App, Arg};
|
||||
|
||||
use crate::{sled::SledDB, sqlite::SqliteDB};
|
||||
|
||||
use itertools::Itertools;
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let matches = App::new("Conduit Sled to Sqlite Migrator")
|
||||
.arg(
|
||||
Arg::with_name("DIR")
|
||||
.long_help("Sets the directory to grab the database from\nWill default to \".\"")
|
||||
.index(1),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("NEW_DIR")
|
||||
.long_help("Sets the destination directory\nWill default to DIR")
|
||||
.index(2),
|
||||
)
|
||||
.get_matches();
|
||||
|
||||
let source_dir = matches.value_of("DIR").unwrap_or(".");
|
||||
|
||||
let dest_dir = matches.value_of("NEW_DIR");
|
||||
|
||||
let source_dir = Path::new(source_dir).canonicalize()?;
|
||||
|
||||
if !source_dir.is_dir() {
|
||||
return Err(anyhow::anyhow!("source path must be directory"));
|
||||
}
|
||||
|
||||
let dest_dir = match dest_dir {
|
||||
None => Ok(source_dir.clone()),
|
||||
Some(dir) => {
|
||||
let p = Path::new(dir).canonicalize()?;
|
||||
if !p.is_dir() {
|
||||
Err(anyhow::anyhow!("destination path must be directory"))
|
||||
} else {
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
}?;
|
||||
|
||||
dbg!(&source_dir, &dest_dir);
|
||||
|
||||
let sled = SledDB::new(crate::sled::new_db(source_dir)?);
|
||||
|
||||
let mut sqlite = SqliteDB::new(sqlite::new_conn(dest_dir)?);
|
||||
|
||||
for (tree, i) in sled.iter() {
|
||||
let tree = String::from_utf8(tree)?;
|
||||
|
||||
dbg!(&tree);
|
||||
|
||||
let mut t = sqlite.table(tree)?;
|
||||
|
||||
let mut x: u32 = 0;
|
||||
|
||||
for chunk in &i.chunks(1000) {
|
||||
dbg!(&x);
|
||||
t.batch_insert(chunk)?;
|
||||
x += 1000;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
44
tools/sled_to_sqlite/src/sled.rs
Normal file
44
tools/sled_to_sqlite/src/sled.rs
Normal file
|
@ -0,0 +1,44 @@
|
|||
use std::path::Path;
|
||||
|
||||
use sled;
|
||||
|
||||
pub fn new_db<P: AsRef<Path>>(path: P) -> sled::Result<sled::Db> {
|
||||
sled::Config::default()
|
||||
.path(path)
|
||||
.use_compression(true)
|
||||
.open()
|
||||
}
|
||||
|
||||
pub struct SledDB(sled::Db);
|
||||
|
||||
impl SledDB {
|
||||
pub fn iter<'a>(
|
||||
&'a self,
|
||||
) -> impl Iterator<Item = (Vec<u8>, impl Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a)> {
|
||||
self.0
|
||||
.tree_names()
|
||||
.into_iter()
|
||||
.map(|v| v.to_vec())
|
||||
.filter_map(move |v| {
|
||||
let t = if let Ok(t) = self.0.open_tree(&v) {
|
||||
t
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let i = t.into_iter().filter_map(|r| {
|
||||
if let Ok(t) = r {
|
||||
Some((t.0.to_vec(), t.1.to_vec()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
Some((v, i))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn new(db: sled::Db) -> Self {
|
||||
Self(db)
|
||||
}
|
||||
}
|
46
tools/sled_to_sqlite/src/sqlite.rs
Normal file
46
tools/sled_to_sqlite/src/sqlite.rs
Normal file
|
@ -0,0 +1,46 @@
|
|||
use std::path::Path;
|
||||
|
||||
use rusqlite::{self, Connection, DatabaseName::Main};
|
||||
|
||||
pub fn new_conn<P: AsRef<Path>>(path: P) -> rusqlite::Result<Connection> {
|
||||
let path = path.as_ref().join("conduit.db");
|
||||
let conn = Connection::open(path)?;
|
||||
|
||||
conn.pragma_update(Some(Main), "journal_mode", &"WAL".to_owned())?;
|
||||
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
pub struct SqliteDB(Connection);
|
||||
|
||||
impl SqliteDB {
|
||||
pub fn new(conn: Connection) -> Self {
|
||||
Self(conn)
|
||||
}
|
||||
|
||||
pub fn table<'a>(&'a mut self, string: String) -> rusqlite::Result<SqliteTable<'a>> {
|
||||
// taken from src/database/abstraction/sqlite.rs
|
||||
self.0.execute(format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", &string).as_str(), [])?;
|
||||
|
||||
Ok(SqliteTable(&mut self.0, string))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SqliteTable<'a>(&'a mut Connection, String);
|
||||
|
||||
impl SqliteTable<'_> {
|
||||
pub fn batch_insert(
|
||||
&mut self,
|
||||
batch: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
|
||||
) -> rusqlite::Result<()> {
|
||||
let tx = self.0.transaction()?;
|
||||
let sql_s = format!("INSERT INTO {} (key, value) VALUES (?, ?)", &self.1);
|
||||
let sql = sql_s.as_str();
|
||||
|
||||
for (k, v) in batch {
|
||||
tx.execute(sql, rusqlite::params![k, v])?;
|
||||
}
|
||||
|
||||
tx.commit()
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue