This commit is contained in:
Jonathan de Jong 2021-07-13 16:44:33 +02:00
parent 03fde79046
commit 8eabefe55b
9 changed files with 159 additions and 82 deletions

View file

@ -6,8 +6,6 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
sled = { version = "0.34.6", features = ["compression", "no_metrics"] }
rusqlite = { version = "0.25.3", features = ["bundled"] }
clap = "2.33.3"
anyhow = "1.0.41"
itertools = "0.10.1"
conduit_iface = { path = "../iface/" }

View file

@ -1,12 +1,8 @@
mod sled;
mod sqlite;
use std::path::Path;
use clap::{App, Arg};
use crate::{sled::SledDB, sqlite::SqliteDB};
use itertools::Itertools;
use conduit_iface::db::{copy_database, sled, sqlite};
fn main() -> anyhow::Result<()> {
let matches = App::new("Conduit Sled to Sqlite Migrator")
@ -46,25 +42,11 @@ fn main() -> anyhow::Result<()> {
dbg!(&source_dir, &dest_dir);
let sled = SledDB::new(crate::sled::new_db(source_dir)?);
let sled = sled::SledDB::new(sled::new_db(source_dir)?);
let mut sqlite = SqliteDB::new(sqlite::new_conn(dest_dir)?);
let mut sqlite = sqlite::SqliteDB::new(sqlite::new_conn(dest_dir)?);
for (tree, i) in sled.iter() {
let tree = String::from_utf8(tree)?;
dbg!(&tree);
let mut t = sqlite.table(tree)?;
let mut x: u32 = 0;
for chunk in &i.chunks(1000) {
dbg!(&x);
t.batch_insert(chunk)?;
x += 1000;
}
}
copy_database(&sled, &mut sqlite, 1000)?;
Ok(())
}

View file

@ -1,44 +0,0 @@
use std::path::Path;
use sled;
pub fn new_db<P: AsRef<Path>>(path: P) -> sled::Result<sled::Db> {
sled::Config::default()
.path(path)
.use_compression(true)
.open()
}
pub struct SledDB(sled::Db);
impl SledDB {
pub fn iter<'a>(
&'a self,
) -> impl Iterator<Item = (Vec<u8>, impl Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a)> {
self.0
.tree_names()
.into_iter()
.map(|v| v.to_vec())
.filter_map(move |v| {
let t = if let Ok(t) = self.0.open_tree(&v) {
t
} else {
return None;
};
let i = t.into_iter().filter_map(|r| {
if let Ok(t) = r {
Some((t.0.to_vec(), t.1.to_vec()))
} else {
None
}
});
Some((v, i))
})
}
pub fn new(db: sled::Db) -> Self {
Self(db)
}
}

View file

@ -1,46 +0,0 @@
use std::path::Path;
use rusqlite::{self, Connection, DatabaseName::Main};
pub fn new_conn<P: AsRef<Path>>(path: P) -> rusqlite::Result<Connection> {
let path = path.as_ref().join("conduit.db");
let conn = Connection::open(path)?;
conn.pragma_update(Some(Main), "journal_mode", &"WAL".to_owned())?;
Ok(conn)
}
pub struct SqliteDB(Connection);
impl SqliteDB {
pub fn new(conn: Connection) -> Self {
Self(conn)
}
pub fn table<'a>(&'a mut self, string: String) -> rusqlite::Result<SqliteTable<'a>> {
// taken from src/database/abstraction/sqlite.rs
self.0.execute(format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", &string).as_str(), [])?;
Ok(SqliteTable(&mut self.0, string))
}
}
pub struct SqliteTable<'a>(&'a mut Connection, String);
impl SqliteTable<'_> {
pub fn batch_insert(
&mut self,
batch: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
) -> rusqlite::Result<()> {
let tx = self.0.transaction()?;
let sql_s = format!("INSERT INTO {} (key, value) VALUES (?, ?)", &self.1);
let sql = sql_s.as_str();
for (k, v) in batch {
tx.execute(sql, rusqlite::params![k, v])?;
}
tx.commit()
}
}