fix-gloss

db/src/db.rs

@@ -76,12 +76,19 @@ pub fn insert_word (
     entry : serde_json::Value,
 ) -> Option<()> {
     use crate::schema::words;
+    let word = entry_as_word (&entry)?;
     let new_word = NewWord {
-        word: entry_as_word (&entry)?,
+        word,
         info: &entry.to_string (),
     };
     diesel::insert_into (words::table)
         .values (new_word)
-        .execute (conn).ok ()?;
+        .execute (conn).map_or_else (
+            |e| {
+                println! ("error for {}: {}", word, e);
+                None
+            },
+            Some
+        )?;
     Some (())
 }

db/src/main.rs

@@ -2,6 +2,8 @@ pub mod models;
 pub mod schema;
 pub mod db;
 
+use diesel::SqliteConnection;
+use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
 use flate2::read::GzDecoder;
 use indicatif::{ProgressBar, ProgressStyle};
 use indicatif::ParallelProgressIterator;
@@ -33,6 +35,18 @@ fn open_wiktextract_dump (path : PathBuf) -> Result<
 #[allow(nonstandard_style)]
 const wiktextract_dump_lines : u64 = 10736399;
 
+fn do_line (
+    conn: &mut PooledConnection<ConnectionManager<SqliteConnection>>,
+    retries : &RwLock<Vec<String>>,
+    line : String
+) {
+    if let Ok (word) = serde_json::from_str::<Value> (&line) {
+        if let None = db::insert_word (conn, word) {
+            retries.write ().unwrap ().push (line);
+        }
+    }
+}
+
 fn main() -> Result<(), String> {
     let args = Args::parse ();
     let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
@@ -46,15 +60,21 @@ fn main() -> Result<(), String> {
         .par_bridge ()
         .progress_with (bar);
     let pool = db::get_pool ();
+    let retries = RwLock::new (vec! []);
 
     lines.for_each (|line| {
         let conn = &mut pool.clone ().get ().unwrap ();
-        if let Ok (word) = serde_json::from_str::<Value> (
-            &line
-        ) {
+        do_line (conn, &retries, line)
+    });
+
+    println! ("do retries now");
+    let conn = &mut pool.clone ().get ().unwrap ();
+    for line in retries.read ().unwrap ().iter () {
+        println! ("retrying");
+        if let Ok (word) = serde_json::from_str::<Value> (&line) {
             db::insert_word (conn, word);
         }
-    });
+    }
 
     Ok (())
 }

src/wiktionary_tf2/main.clj

@@ -22,15 +22,19 @@
   (when-some [[_ word gloss] (re-matches #"([^\t]*)\t(.*)\n?" s)]
     [word gloss]))
 
-(defn lookup-word [word]
+(defn query-word [word]
   (let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})]
     (with-open [conn (sql/get-connection ds)]
       (->> (sql/execute! conn ["select info from words where word = ?"
                                word])
-           (map #(json/parse-string (:words/info %) keyword))
-           (mapcat :senses)
-           (mapcat :raw_glosses)
-           first))))
+           (map #(json/parse-string (:words/info %) keyword))))))
+
+(defn gloss-word [word]
+  (->> word
+       query-word
+       (mapcat :senses)
+       (mapcat (some-fn :raw_glosses :glosses))
+       first))
 
 (defn parse-chat-message [x]
   (when-let [[_ _dead? author body]
@@ -52,7 +56,7 @@
 (defn do-definition [s]
   (when-some [w (find-word s)]
     (l/infof "looking up word: %s" w)
-    (let [r (lookup-word w)]
+    (let [r (gloss-word w)]
       (if r
         (l/infof "found word! %s" r)
         (l/infof "no entry... %s" w))