From 31ce64ea72da624e2a3b546144dcadf314a16ab2 Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Tue, 25 Jul 2023 16:05:32 -0700 Subject: [PATCH 1/2] discard broken sqlite rows I did some database surgery on my SQLite DB which somehow caused some rows to be messed up. This change just discords broken rows instead of panicking, allowing the migration to finish. --- tools/iface/src/db/sqlite.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/iface/src/db/sqlite.rs b/tools/iface/src/db/sqlite.rs index 458e25b..fe9eb01 100644 --- a/tools/iface/src/db/sqlite.rs +++ b/tools/iface/src/db/sqlite.rs @@ -106,9 +106,22 @@ impl SegmentIter for SqliteSegmentIter<'_> { fn iter<'f>(&'f mut self) -> KVIter<'f> { Box::new( self.0 - .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) + .query_map([], |row| Ok((row.get(0), row.get(1)))) .unwrap() - .map(|r| r.unwrap()), + .map(|x| x.unwrap()) + .filter_map(|(k, v)| { + let Ok(k) = k else { + println!("ignored a row because its key is malformed"); + return None; + }; + + let Ok(v) = v else { + println!("ignored a row because its value is malformed"); + return None; + }; + + Some((k, v)) + }), ) } } From 4a48ab3b503461d13bf4d7bf613381dfe1a36d4e Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Thu, 27 Jul 2023 11:35:07 -0700 Subject: [PATCH 2/2] only ignore broken rows upon request --- tools/iface/src/db.rs | 5 +++++ tools/iface/src/db/sqlite.rs | 41 ++++++++++++++++++++++++++---------- tools/migrate/src/main.rs | 16 +++++++++++--- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/tools/iface/src/db.rs b/tools/iface/src/db.rs index e695f10..a6b501a 100644 --- a/tools/iface/src/db.rs +++ b/tools/iface/src/db.rs @@ -15,6 +15,11 @@ pub type KVIter<'a> = Box, Vec)> + 'a>; pub type TreeKVIter<'a> = Box, KVIter<'a>)> + 'a>; +#[derive(Clone, Copy)] +pub struct Config { + ignore_broken_rows: bool, +} + pub trait Database { fn names<'a>(&'a self) -> Vec>; diff --git a/tools/iface/src/db/sqlite.rs b/tools/iface/src/db/sqlite.rs index fe9eb01..d5ce752 100644 --- a/tools/iface/src/db/sqlite.rs +++ b/tools/iface/src/db/sqlite.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use rusqlite::{self, Connection, DatabaseName::Main, Statement}; use std::{collections::HashSet, iter::FromIterator, path::Path}; -use super::{Database, KVIter, Segment, SegmentIter}; +use super::{Config, Database, KVIter, Segment, SegmentIter}; pub fn new_conn>(path: P) -> rusqlite::Result { let path = path.as_ref().join("conduit.db"); @@ -15,13 +15,14 @@ pub fn new_conn>(path: P) -> rusqlite::Result { pub struct SqliteDB { conn: Connection, + config: Config, } const CORRECT_TABLE_SET: &[&str] = &["key", "value"]; impl<'a> SqliteDB { - pub fn new(conn: Connection) -> Self { - Self { conn } + pub fn new(conn: Connection, config: Config) -> Self { + Self { conn, config } } fn valid_tables(&self) -> Vec { @@ -62,6 +63,7 @@ impl Database for SqliteDB { Some(Box::new(SqliteSegment { conn: &mut self.conn, name: string, + config: self.config, })) } @@ -73,6 +75,7 @@ impl Database for SqliteDB { pub struct SqliteSegment<'a> { conn: &'a mut Connection, name: String, + config: Config, } impl Segment for SqliteSegment<'_> { @@ -92,31 +95,47 @@ impl Segment for SqliteSegment<'_> { } fn get_iter(&mut self) -> Box { - Box::new(SqliteSegmentIter( - self.conn + Box::new(SqliteSegmentIter { + statement: self + .conn .prepare(format!("SELECT key, value FROM {}", self.name).as_str()) .unwrap(), - )) + config: self.config, + }) } } -struct SqliteSegmentIter<'a>(Statement<'a>); +struct SqliteSegmentIter<'a> { + statement: Statement<'a>, + config: Config, +} impl SegmentIter for SqliteSegmentIter<'_> { fn iter<'f>(&'f mut self) -> KVIter<'f> { + let config = self.config; + Box::new( - self.0 + self.statement .query_map([], |row| Ok((row.get(0), row.get(1)))) .unwrap() .map(|x| x.unwrap()) - .filter_map(|(k, v)| { + .filter_map(move |(k, v)| { + let advice = "You could try using `--ignore-broken-rows` to complete the migration, but take note of its caveats."; let Ok(k) = k else { - println!("ignored a row because its key is malformed"); + if config.ignore_broken_rows { + println!("ignored a row because its key is malformed"); + } else { + panic!("This row has a malformed key. {}", advice); + } return None; }; let Ok(v) = v else { - println!("ignored a row because its value is malformed"); + if config.ignore_broken_rows { + println!("ignored a row because its value is malformed"); + } else { + panic!("This row has a malformed value. {}", advice); + } return None; }; diff --git a/tools/migrate/src/main.rs b/tools/migrate/src/main.rs index 9d1de77..911fe01 100644 --- a/tools/migrate/src/main.rs +++ b/tools/migrate/src/main.rs @@ -1,5 +1,5 @@ use clap::{App, Arg}; -use conduit_iface::db::{self, copy_database}; +use conduit_iface::db::{self, copy_database, Config}; use std::{ ops::{Deref, DerefMut}, path::{Path, PathBuf}, @@ -19,14 +19,17 @@ enum Database { } impl Database { - fn new(name: &str, path: PathBuf) -> anyhow::Result { + fn new(name: &str, path: PathBuf, config: Config) -> anyhow::Result { Ok(match name { #[cfg(feature = "sled")] "sled" => Self::Sled(db::sled::SledDB::new(db::sled::new_db(path)?)), #[cfg(feature = "heed")] "heed" => Self::Heed(db::heed::HeedDB::new(db::heed::new_db(path)?)), #[cfg(feature = "sqlite")] - "sqlite" => Self::Sqlite(db::sqlite::SqliteDB::new(db::sqlite::new_conn(path)?)), + "sqlite" => Self::Sqlite(db::sqlite::SqliteDB::new( + db::sqlite::new_conn(path)?, + config, + )), #[cfg(feature = "rocksdb")] "rocks" => Self::Rocks(db::rocksdb::new_conn(path)?), #[cfg(feature = "persy")] @@ -129,6 +132,11 @@ fn main() -> anyhow::Result<()> { .takes_value(true) .required(true), ) + .arg( + Arg::with_name("ignore_broken_rows") + .long("ignore-broken-rows") + .long_help("Lossy migration methodology if parts of the database are malformed due to e.g. improper manual database surgery. Currently only applies to SQLite.") + ) .get_matches(); let src_dir = matches.value_of("from_dir").unwrap_or("."); @@ -155,6 +163,8 @@ fn main() -> anyhow::Result<()> { dbg!(&src_dir, &dst_dir); + let ignore_broken_rows = matches.is_present("ignore_broken_rows"); + let mut src_db = Database::new(matches.value_of("from").unwrap(), src_dir)?; let mut dst_db = Database::new(matches.value_of("to").unwrap(), dst_dir)?;