fix-gloss

db/src/db.rs

@@ -76,12 +76,19 @@ pub fn insert_word (
     entry : serde_json::Value,
 ) -> Option<()> {
     use crate::schema::words;
+    let word = entry_as_word (&entry)?;
     let new_word = NewWord {
-        word: entry_as_word (&entry)?,
+        word,
         info: &entry.to_string (),
     };
     diesel::insert_into (words::table)
         .values (new_word)
-        .execute (conn).ok ()?;
+        .execute (conn).map_or_else (
+            |e| {
+                println! ("error for {}: {}", word, e);
+                None
+            },
+            Some
+        )?;
     Some (())
 }

db/src/main.rs

@@ -2,6 +2,8 @@ pub mod models;
 pub mod schema;
 pub mod db;
 
+use diesel::SqliteConnection;
+use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
 use flate2::read::GzDecoder;
 use indicatif::{ProgressBar, ProgressStyle};
 use indicatif::ParallelProgressIterator;
@@ -33,6 +35,18 @@ fn open_wiktextract_dump (path : PathBuf) -> Result<
 #[allow(nonstandard_style)]
 const wiktextract_dump_lines : u64 = 10736399;
 
+fn do_line (
+    conn: &mut PooledConnection<ConnectionManager<SqliteConnection>>,
+    retries : &RwLock<Vec<String>>,
+    line : String
+) {
+    if let Ok (word) = serde_json::from_str::<Value> (&line) {
+        if let None = db::insert_word (conn, word) {
+            retries.write ().unwrap ().push (line);
+        }
+    }
+}
+
 fn main() -> Result<(), String> {
     let args = Args::parse ();
     let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
@@ -46,15 +60,21 @@ fn main() -> Result<(), String> {
         .par_bridge ()
         .progress_with (bar);
     let pool = db::get_pool ();
+    let retries = RwLock::new (vec! []);
 
     lines.for_each (|line| {
         let conn = &mut pool.clone ().get ().unwrap ();
-        if let Ok (word) = serde_json::from_str::<Value> (
-            &line
-        ) {
+        do_line (conn, &retries, line)
+    });
+
+    println! ("do retries now");
+    let conn = &mut pool.clone ().get ().unwrap ();
+    for line in retries.read ().unwrap ().iter () {
+        println! ("retrying");
+        if let Ok (word) = serde_json::from_str::<Value> (&line) {
             db::insert_word (conn, word);
         }
-    });
+    }
 
     Ok (())
 }