INIT

tools/sled_to_sqlite/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "conduit_sled_to_sqlite"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+sled = { version = "0.34.6", features = ["compression", "no_metrics"] }
+rusqlite = { version = "0.25.3", features = ["bundled"] }
+clap = "2.33.3"
+anyhow = "1.0.41"
+itertools = "0.10.1"

tools/sled_to_sqlite/src/main.rs

@@ -0,0 +1,70 @@
+mod sled;
+mod sqlite;
+use std::path::Path;
+
+use clap::{App, Arg};
+
+use crate::{sled::SledDB, sqlite::SqliteDB};
+
+use itertools::Itertools;
+
+fn main() -> anyhow::Result<()> {
+    let matches = App::new("Conduit Sled to Sqlite Migrator")
+        .arg(
+            Arg::with_name("DIR")
+                .long_help("Sets the directory to grab the database from\nWill default to \".\"")
+                .index(1),
+        )
+        .arg(
+            Arg::with_name("NEW_DIR")
+                .long_help("Sets the destination directory\nWill default to DIR")
+                .index(2),
+        )
+        .get_matches();
+
+    let source_dir = matches.value_of("DIR").unwrap_or(".");
+
+    let dest_dir = matches.value_of("NEW_DIR");
+
+    let source_dir = Path::new(source_dir).canonicalize()?;
+
+    if !source_dir.is_dir() {
+        return Err(anyhow::anyhow!("source path must be directory"));
+    }
+
+    let dest_dir = match dest_dir {
+        None => Ok(source_dir.clone()),
+        Some(dir) => {
+            let p = Path::new(dir).canonicalize()?;
+            if !p.is_dir() {
+                Err(anyhow::anyhow!("destination path must be directory"))
+            } else {
+                Ok(p)
+            }
+        }
+    }?;
+
+    dbg!(&source_dir, &dest_dir);
+
+    let sled = SledDB::new(crate::sled::new_db(source_dir)?);
+
+    let mut sqlite = SqliteDB::new(sqlite::new_conn(dest_dir)?);
+
+    for (tree, i) in sled.iter() {
+        let tree = String::from_utf8(tree)?;
+
+        dbg!(&tree);
+
+        let mut t = sqlite.table(tree)?;
+
+        let mut x: u32 = 0;
+
+        for chunk in &i.chunks(1000) {
+            dbg!(&x);
+            t.batch_insert(chunk)?;
+            x += 1000;
+        }
+    }
+
+    Ok(())
+}

tools/sled_to_sqlite/src/sled.rs

@@ -0,0 +1,44 @@
+use std::path::Path;
+
+use sled;
+
+pub fn new_db<P: AsRef<Path>>(path: P) -> sled::Result<sled::Db> {
+    sled::Config::default()
+        .path(path)
+        .use_compression(true)
+        .open()
+}
+
+pub struct SledDB(sled::Db);
+
+impl SledDB {
+    pub fn iter<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = (Vec<u8>, impl Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a)> {
+        self.0
+            .tree_names()
+            .into_iter()
+            .map(|v| v.to_vec())
+            .filter_map(move |v| {
+                let t = if let Ok(t) = self.0.open_tree(&v) {
+                    t
+                } else {
+                    return None;
+                };
+
+                let i = t.into_iter().filter_map(|r| {
+                    if let Ok(t) = r {
+                        Some((t.0.to_vec(), t.1.to_vec()))
+                    } else {
+                        None
+                    }
+                });
+
+                Some((v, i))
+            })
+    }
+
+    pub fn new(db: sled::Db) -> Self {
+        Self(db)
+    }
+}

tools/sled_to_sqlite/src/sqlite.rs

@@ -0,0 +1,46 @@
+use std::path::Path;
+
+use rusqlite::{self, Connection, DatabaseName::Main};
+
+pub fn new_conn<P: AsRef<Path>>(path: P) -> rusqlite::Result<Connection> {
+    let path = path.as_ref().join("conduit.db");
+    let conn = Connection::open(path)?;
+
+    conn.pragma_update(Some(Main), "journal_mode", &"WAL".to_owned())?;
+
+    Ok(conn)
+}
+
+pub struct SqliteDB(Connection);
+
+impl SqliteDB {
+    pub fn new(conn: Connection) -> Self {
+        Self(conn)
+    }
+
+    pub fn table<'a>(&'a mut self, string: String) -> rusqlite::Result<SqliteTable<'a>> {
+        // taken from src/database/abstraction/sqlite.rs
+        self.0.execute(format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", &string).as_str(), [])?;
+
+        Ok(SqliteTable(&mut self.0, string))
+    }
+}
+
+pub struct SqliteTable<'a>(&'a mut Connection, String);
+
+impl SqliteTable<'_> {
+    pub fn batch_insert(
+        &mut self,
+        batch: impl Iterator<Item = (Vec<u8>, Vec<u8>)>,
+    ) -> rusqlite::Result<()> {
+        let tx = self.0.transaction()?;
+        let sql_s = format!("INSERT INTO {} (key, value) VALUES (?, ?)", &self.1);
+        let sql = sql_s.as_str();
+
+        for (k, v) in batch {
+            tx.execute(sql, rusqlite::params![k, v])?;
+        }
+
+        tx.commit()
+    }
+}