mirror of
https://github.com/ShadowJonathan/conduit_toolbox.git
synced 2025-07-26 13:24:07 +03:00
only ignore broken rows upon request
This commit is contained in:
parent
31ce64ea72
commit
4a48ab3b50
3 changed files with 48 additions and 14 deletions
|
@ -15,6 +15,11 @@ pub type KVIter<'a> = Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
|
|||
|
||||
pub type TreeKVIter<'a> = Box<dyn Iterator<Item = (Vec<u8>, KVIter<'a>)> + 'a>;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Config {
|
||||
ignore_broken_rows: bool,
|
||||
}
|
||||
|
||||
pub trait Database {
|
||||
fn names<'a>(&'a self) -> Vec<Vec<u8>>;
|
||||
|
||||
|
|
|
@ -2,7 +2,7 @@ use itertools::Itertools;
|
|||
use rusqlite::{self, Connection, DatabaseName::Main, Statement};
|
||||
use std::{collections::HashSet, iter::FromIterator, path::Path};
|
||||
|
||||
use super::{Database, KVIter, Segment, SegmentIter};
|
||||
use super::{Config, Database, KVIter, Segment, SegmentIter};
|
||||
|
||||
pub fn new_conn<P: AsRef<Path>>(path: P) -> rusqlite::Result<Connection> {
|
||||
let path = path.as_ref().join("conduit.db");
|
||||
|
@ -15,13 +15,14 @@ pub fn new_conn<P: AsRef<Path>>(path: P) -> rusqlite::Result<Connection> {
|
|||
|
||||
pub struct SqliteDB {
|
||||
conn: Connection,
|
||||
config: Config,
|
||||
}
|
||||
|
||||
const CORRECT_TABLE_SET: &[&str] = &["key", "value"];
|
||||
|
||||
impl<'a> SqliteDB {
|
||||
pub fn new(conn: Connection) -> Self {
|
||||
Self { conn }
|
||||
pub fn new(conn: Connection, config: Config) -> Self {
|
||||
Self { conn, config }
|
||||
}
|
||||
|
||||
fn valid_tables(&self) -> Vec<String> {
|
||||
|
@ -62,6 +63,7 @@ impl Database for SqliteDB {
|
|||
Some(Box::new(SqliteSegment {
|
||||
conn: &mut self.conn,
|
||||
name: string,
|
||||
config: self.config,
|
||||
}))
|
||||
}
|
||||
|
||||
|
@ -73,6 +75,7 @@ impl Database for SqliteDB {
|
|||
pub struct SqliteSegment<'a> {
|
||||
conn: &'a mut Connection,
|
||||
name: String,
|
||||
config: Config,
|
||||
}
|
||||
|
||||
impl Segment for SqliteSegment<'_> {
|
||||
|
@ -92,31 +95,47 @@ impl Segment for SqliteSegment<'_> {
|
|||
}
|
||||
|
||||
fn get_iter(&mut self) -> Box<dyn super::SegmentIter + '_> {
|
||||
Box::new(SqliteSegmentIter(
|
||||
self.conn
|
||||
Box::new(SqliteSegmentIter {
|
||||
statement: self
|
||||
.conn
|
||||
.prepare(format!("SELECT key, value FROM {}", self.name).as_str())
|
||||
.unwrap(),
|
||||
))
|
||||
config: self.config,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct SqliteSegmentIter<'a>(Statement<'a>);
|
||||
struct SqliteSegmentIter<'a> {
|
||||
statement: Statement<'a>,
|
||||
config: Config,
|
||||
}
|
||||
|
||||
impl SegmentIter for SqliteSegmentIter<'_> {
|
||||
fn iter<'f>(&'f mut self) -> KVIter<'f> {
|
||||
let config = self.config;
|
||||
|
||||
Box::new(
|
||||
self.0
|
||||
self.statement
|
||||
.query_map([], |row| Ok((row.get(0), row.get(1))))
|
||||
.unwrap()
|
||||
.map(|x| x.unwrap())
|
||||
.filter_map(|(k, v)| {
|
||||
.filter_map(move |(k, v)| {
|
||||
let advice = "You could try using `--ignore-broken-rows` to complete the migration, but take note of its caveats.";
|
||||
let Ok(k) = k else {
|
||||
println!("ignored a row because its key is malformed");
|
||||
if config.ignore_broken_rows {
|
||||
println!("ignored a row because its key is malformed");
|
||||
} else {
|
||||
panic!("This row has a malformed key. {}", advice);
|
||||
}
|
||||
return None;
|
||||
};
|
||||
|
||||
let Ok(v) = v else {
|
||||
println!("ignored a row because its value is malformed");
|
||||
if config.ignore_broken_rows {
|
||||
println!("ignored a row because its value is malformed");
|
||||
} else {
|
||||
panic!("This row has a malformed value. {}", advice);
|
||||
}
|
||||
return None;
|
||||
};
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue