81 lines
2.3 KiB
Rust
81 lines
2.3 KiB
Rust
pub mod models;
|
|
pub mod schema;
|
|
pub mod db;
|
|
|
|
use diesel::SqliteConnection;
|
|
use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
|
|
use flate2::read::GzDecoder;
|
|
use indicatif::{ProgressBar, ProgressStyle};
|
|
use indicatif::ParallelProgressIterator;
|
|
use serde_json::Value;
|
|
use wiktionary_schema::en::WordData;
|
|
use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
|
|
use clap::Parser;
|
|
use rayon::prelude::*;
|
|
|
|
/// Build an SQLite database from a Wiktextract JSONLines dump.
|
|
#[derive(Parser, Debug)]
|
|
#[command(name = "make-db", version, about, long_about = None)]
|
|
struct Args {
|
|
/// Input file
|
|
#[clap(long, value_parser)]
|
|
wiktextract_dump : PathBuf,
|
|
|
|
/// Output file
|
|
#[clap(long, value_parser)]
|
|
database : Option<PathBuf>,
|
|
}
|
|
|
|
fn open_wiktextract_dump (path : PathBuf) -> Result<
|
|
BufReader<GzDecoder<File>>, io::Error
|
|
> {
|
|
Ok (BufReader::new (GzDecoder::new (File::open (path)?)))
|
|
}
|
|
|
|
#[allow(nonstandard_style)]
|
|
const wiktextract_dump_lines : u64 = 10736399;
|
|
|
|
fn do_line (
|
|
conn: &mut PooledConnection<ConnectionManager<SqliteConnection>>,
|
|
retries : &RwLock<Vec<String>>,
|
|
line : String
|
|
) {
|
|
if let Ok (word) = serde_json::from_str::<Value> (&line) {
|
|
if let None = db::insert_word (conn, word) {
|
|
retries.write ().unwrap ().push (line);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn main() -> Result<(), String> {
|
|
let args = Args::parse ();
|
|
let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
|
|
let bar = ProgressBar::new (wiktextract_dump_lines);
|
|
bar.set_style (
|
|
ProgressStyle::with_template(
|
|
"{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
|
|
) .unwrap()
|
|
);
|
|
let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
|
|
.par_bridge ()
|
|
.progress_with (bar);
|
|
let pool = db::get_pool ();
|
|
let retries = RwLock::new (vec! []);
|
|
|
|
lines.for_each (|line| {
|
|
let conn = &mut pool.clone ().get ().unwrap ();
|
|
do_line (conn, &retries, line)
|
|
});
|
|
|
|
println! ("do retries now");
|
|
let conn = &mut pool.clone ().get ().unwrap ();
|
|
for line in retries.read ().unwrap ().iter () {
|
|
println! ("retrying");
|
|
if let Ok (word) = serde_json::from_str::<Value> (&line) {
|
|
db::insert_word (conn, word);
|
|
}
|
|
}
|
|
|
|
Ok (())
|
|
}
|