make-db

db/src/db.rs

@@ -0,0 +1,45 @@
+use diesel::prelude::*;
+use std::env;
+use crate::models::*;
+
+pub fn connect () -> SqliteConnection {
+    let database_url = env::var ("DATABASE_URL")
+        .expect ("DATABASE_URL must be set");
+    SqliteConnection::establish (&database_url)
+        .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url))
+}
+
+pub fn get_word (w : &str) -> Vec<Word> {
+    let conn = &mut connect ();
+    use crate::schema::words::dsl::*;
+    let results = words
+        .filter (word.eq (w))
+        .limit (5)
+        .select (Word::as_select ())
+        .load (conn)
+        .expect ("error loading words");
+    for wd in results {
+        println! ("{}", wd.word);
+    }
+    todo! ()
+}
+
+pub fn entry_as_word (entry : &serde_json::Value) -> Option<&str> {
+    entry.get ("word")?.as_str ()
+}
+
+pub fn insert_word (
+    conn : &mut SqliteConnection,
+    entry : serde_json::Value,
+) -> Option<()> {
+    use crate::schema::words;
+    let new_word = NewWord {
+        word: entry_as_word (&entry)?,
+        info: &entry.to_string (),
+    };
+    diesel::insert_into (words::table)
+        .values (new_word)
+        .execute (conn)
+        .expect ("error saving new word");
+    Some (())
+}

db/src/main.rs

@@ -0,0 +1,52 @@
+pub mod models;
+pub mod schema;
+pub mod db;
+
+use flate2::read::GzDecoder;
+use indicatif::ProgressBar;
+use serde_json::Value;
+use wiktionary_schema::en::WordData;
+use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf};
+use clap::Parser;
+
+/// Build an SQLite database from a Wiktextract JSONLines dump.
+#[derive(Parser, Debug)]
+#[command(name = "make-db", version, about, long_about = None)]
+struct Args {
+    /// Input file
+    #[clap(long, value_parser)]
+    wiktextract_dump : PathBuf,
+
+    /// Output file
+    #[clap(long, value_parser)]
+    database : Option<PathBuf>,
+}
+
+fn open_wiktextract_dump (path : PathBuf) -> Result<
+    BufReader<GzDecoder<File>>, io::Error
+> {
+    Ok (BufReader::new (GzDecoder::new (File::open (path)?)))
+}
+
+#[allow(nonstandard_style)]
+const wiktextract_dump_lines : u64 = 10736399;
+
+fn main() -> Result<(), String> {
+    let args = Args::parse ();
+    let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
+    let bar = ProgressBar::new (wiktextract_dump_lines);
+    let conn = &mut db::connect ();
+
+    for i in d.lines () {
+        if let Ok (word) = serde_json::from_str::<Value> (
+            &i.expect ("entry")
+        ) {
+            bar.println (db::entry_as_word (&word).map_or ("", |x| x));
+            db::insert_word (conn, word);
+            bar.inc (1);
+        }
+    }
+
+    bar.finish ();
+    Ok (())
+}

db/src/models.rs

@@ -0,0 +1,17 @@
+use diesel::prelude::*;
+
+#[derive(Queryable, Selectable)]
+#[diesel(table_name = crate::schema::words)]
+#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
+pub struct Word {
+    pub id : i32,
+    pub word : String,
+    pub info : String,
+}
+
+#[derive(Insertable)]
+#[diesel(table_name = crate::schema::words)]
+pub struct NewWord<'a> {
+    pub word : &'a str,
+    pub info : &'a str,
+}

db/src/schema.rs

@@ -0,0 +1,9 @@
+// @generated automatically by Diesel CLI.
+
+diesel::table! {
+    words (id) {
+        id -> Integer,
+        word -> Text,
+        info -> Text,
+    }
+}