refactor

tools/iface/src/db.rs

@@ -0,0 +1,43 @@
+pub mod sled;
+pub mod sqlite;
+
+use itertools::Itertools;
+
+pub type KVIter<'a> = Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
+
+pub type TreeKVIter<'a> = Box<dyn Iterator<Item = (Vec<u8>, KVIter<'a>)> + 'a>;
+
+pub trait Database {
+    fn iter<'a>(&'a self) -> TreeKVIter<'a>;
+
+    fn segment<'a>(&'a mut self, name: Vec<u8>) -> Option<Box<dyn Segment + 'a>>; // change return type to Result
+}
+
+pub trait Segment {
+    fn batch_insert<'a>(
+        &'a mut self,
+        batch: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>,
+    ) -> anyhow::Result<()>;
+}
+
+pub fn copy_database(
+    src: &impl Database,
+    dst: &mut impl Database,
+    chunk_size: usize,
+) -> anyhow::Result<()> {
+    for (tree, i) in src.iter() {
+        dbg!(&tree);
+
+        let mut t = dst.segment(tree).unwrap(); // todo remove unwrap
+
+        let mut x: usize = 0;
+
+        for chunk in &i.chunks(chunk_size) {
+            dbg!(&x);
+            t.batch_insert(Box::new(chunk))?;
+            x += chunk_size;
+        }
+    }
+
+    Ok(())
+}

tools/iface/src/db/sled.rs

@@ -0,0 +1,68 @@
+use std::path::Path;
+
+use sled::{Batch, Config, Db, Result, Tree};
+
+use super::{Database, KVIter, Segment, TreeKVIter};
+
+pub fn new_db<P: AsRef<Path>>(path: P) -> Result<Db> {
+    Config::default().path(path).use_compression(true).open()
+}
+
+pub struct SledDB(Db);
+
+impl SledDB {
+    pub fn new(db: Db) -> Self {
+        Self(db)
+    }
+}
+
+impl Database for SledDB {
+    fn iter<'a>(&'a self) -> TreeKVIter<'a> {
+        Box::new(
+            self.0
+                .tree_names()
+                .into_iter()
+                .map(|v| v.to_vec())
+                .filter_map(move |v| {
+                    if let Ok(t) = self.0.open_tree(&v) {
+                        Some((v, t))
+                    } else {
+                        None
+                    }
+                })
+                .map(|(v, t): (Vec<u8>, Tree)| -> (Vec<u8>, KVIter<'a>) {
+                    let i = t.into_iter().filter_map(|r| {
+                        if let Ok(t) = r {
+                            Some((t.0.to_vec(), t.1.to_vec()))
+                        } else {
+                            None
+                        }
+                    });
+
+                    (v, Box::new(i))
+                }),
+        )
+    }
+
+    fn segment(&mut self, name: Vec<u8>) -> Option<Box<dyn Segment>> {
+        self.0
+            .open_tree(name)
+            .ok()
+            .map(|t| -> Box<dyn Segment> { Box::new(t) })
+    }
+}
+
+impl Segment for Tree {
+    fn batch_insert(
+        &mut self,
+        batch: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + '_>,
+    ) -> anyhow::Result<()> {
+        let mut sled_batch = Batch::default();
+
+        for (k, v) in batch {
+            sled_batch.insert(k, v)
+        }
+
+        self.apply_batch(sled_batch).map_err(Into::into)
+    }
+}

tools/iface/src/db/sqlite.rs

@@ -0,0 +1,55 @@
+use std::path::Path;
+
+use rusqlite::{self, Connection, DatabaseName::Main};
+
+use super::{Database, Segment};
+
+pub fn new_conn<P: AsRef<Path>>(path: P) -> rusqlite::Result<Connection> {
+    let path = path.as_ref().join("conduit.db");
+    let conn = Connection::open(path)?;
+
+    conn.pragma_update(Some(Main), "journal_mode", &"WAL".to_owned())?;
+
+    Ok(conn)
+}
+
+pub struct SqliteDB(Connection);
+
+impl SqliteDB {
+    pub fn new(conn: Connection) -> Self {
+        Self(conn)
+    }
+}
+
+impl Database for SqliteDB {
+    fn iter<'a>(&'a self) -> super::TreeKVIter<'a> {
+        todo!("iterate over tables, pick only tables that have columns 'key' and 'value', then iterate over that with values")
+    }
+
+    fn segment<'a>(&'a mut self, name: Vec<u8>) -> Option<Box<dyn Segment + 'a>> {
+        let string = String::from_utf8(name).unwrap();
+        // taken from src/database/abstraction/sqlite.rs
+        self.0.execute(format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", &string).as_str(), []).unwrap();
+
+        Some(Box::new(SqliteTable(&mut self.0, string)))
+    }
+}
+
+pub struct SqliteTable<'a>(&'a mut Connection, String);
+
+impl Segment for SqliteTable<'_> {
+    fn batch_insert(
+        &mut self,
+        batch: Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + '_>,
+    ) -> anyhow::Result<()> {
+        let tx = self.0.transaction()?;
+        let sql_s = format!("INSERT INTO {} (key, value) VALUES (?, ?)", &self.1);
+        let sql = sql_s.as_str();
+
+        for (k, v) in batch {
+            tx.execute(sql, rusqlite::params![k, v])?;
+        }
+
+        tx.commit().map_err(Into::into)
+    }
+}

tools/iface/src/lib.rs

@@ -0,0 +1 @@
+pub mod db;