make-db

db/src/main.rs

@@ -0,0 +1,52 @@
+pub mod models;
+pub mod schema;
+pub mod db;
+
+use flate2::read::GzDecoder;
+use indicatif::ProgressBar;
+use serde_json::Value;
+use wiktionary_schema::en::WordData;
+use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf};
+use clap::Parser;
+
+/// Build an SQLite database from a Wiktextract JSONLines dump.
+#[derive(Parser, Debug)]
+#[command(name = "make-db", version, about, long_about = None)]
+struct Args {
+    /// Input file
+    #[clap(long, value_parser)]
+    wiktextract_dump : PathBuf,
+
+    /// Output file
+    #[clap(long, value_parser)]
+    database : Option<PathBuf>,
+}
+
+fn open_wiktextract_dump (path : PathBuf) -> Result<
+    BufReader<GzDecoder<File>>, io::Error
+> {
+    Ok (BufReader::new (GzDecoder::new (File::open (path)?)))
+}
+
+#[allow(nonstandard_style)]
+const wiktextract_dump_lines : u64 = 10736399;
+
+fn main() -> Result<(), String> {
+    let args = Args::parse ();
+    let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
+    let bar = ProgressBar::new (wiktextract_dump_lines);
+    let conn = &mut db::connect ();
+
+    for i in d.lines () {
+        if let Ok (word) = serde_json::from_str::<Value> (
+            &i.expect ("entry")
+        ) {
+            bar.println (db::entry_as_word (&word).map_or ("", |x| x));
+            db::insert_word (conn, word);
+            bar.inc (1);
+        }
+    }
+
+    bar.finish ();
+    Ok (())
+}