diff --git a/db/flamegraph.svg b/db/flamegraph.svg
deleted file mode 100644
index e5a306a..0000000
--- a/db/flamegraph.svg
+++ /dev/null
@@ -1,491 +0,0 @@
-
\ No newline at end of file
diff --git a/db/src/db.rs b/db/src/db.rs
index 7a7bdf9..6ecf2b6 100644
--- a/db/src/db.rs
+++ b/db/src/db.rs
@@ -76,12 +76,19 @@ pub fn insert_word (
entry : serde_json::Value,
) -> Option<()> {
use crate::schema::words;
+ let word = entry_as_word (&entry)?;
let new_word = NewWord {
- word: entry_as_word (&entry)?,
+ word,
info: &entry.to_string (),
};
diesel::insert_into (words::table)
.values (new_word)
- .execute (conn).ok ()?;
+ .execute (conn).map_or_else (
+ |e| {
+ println! ("error for {}: {}", word, e);
+ None
+ },
+ Some
+ )?;
Some (())
}
diff --git a/db/src/main.rs b/db/src/main.rs
index 18fc983..4526dfb 100644
--- a/db/src/main.rs
+++ b/db/src/main.rs
@@ -2,6 +2,8 @@ pub mod models;
pub mod schema;
pub mod db;
+use diesel::SqliteConnection;
+use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
use flate2::read::GzDecoder;
use indicatif::{ProgressBar, ProgressStyle};
use indicatif::ParallelProgressIterator;
@@ -33,6 +35,18 @@ fn open_wiktextract_dump (path : PathBuf) -> Result<
#[allow(nonstandard_style)]
const wiktextract_dump_lines : u64 = 10736399;
+fn do_line (
+ conn: &mut PooledConnection>,
+ retries : &RwLock>,
+ line : String
+) {
+ if let Ok (word) = serde_json::from_str:: (&line) {
+ if let None = db::insert_word (conn, word) {
+ retries.write ().unwrap ().push (line);
+ }
+ }
+}
+
fn main() -> Result<(), String> {
let args = Args::parse ();
let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
@@ -46,15 +60,21 @@ fn main() -> Result<(), String> {
.par_bridge ()
.progress_with (bar);
let pool = db::get_pool ();
+ let retries = RwLock::new (vec! []);
lines.for_each (|line| {
let conn = &mut pool.clone ().get ().unwrap ();
- if let Ok (word) = serde_json::from_str:: (
- &line
- ) {
+ do_line (conn, &retries, line)
+ });
+
+ println! ("do retries now");
+ let conn = &mut pool.clone ().get ().unwrap ();
+ for line in retries.read ().unwrap ().iter () {
+ println! ("retrying");
+ if let Ok (word) = serde_json::from_str:: (&line) {
db::insert_word (conn, word);
}
- });
+ }
Ok (())
}
diff --git a/src/wiktionary_tf2/main.clj b/src/wiktionary_tf2/main.clj
index 06bf44c..610e433 100644
--- a/src/wiktionary_tf2/main.clj
+++ b/src/wiktionary_tf2/main.clj
@@ -22,15 +22,19 @@
(when-some [[_ word gloss] (re-matches #"([^\t]*)\t(.*)\n?" s)]
[word gloss]))
-(defn lookup-word [word]
+(defn query-word [word]
(let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})]
(with-open [conn (sql/get-connection ds)]
(->> (sql/execute! conn ["select info from words where word = ?"
word])
- (map #(json/parse-string (:words/info %) keyword))
- (mapcat :senses)
- (mapcat :raw_glosses)
- first))))
+ (map #(json/parse-string (:words/info %) keyword))))))
+
+(defn gloss-word [word]
+ (->> word
+ query-word
+ (mapcat :senses)
+ (mapcat (some-fn :raw_glosses :glosses))
+ first))
(defn parse-chat-message [x]
(when-let [[_ _dead? author body]
@@ -52,7 +56,7 @@
(defn do-definition [s]
(when-some [w (find-word s)]
(l/infof "looking up word: %s" w)
- (let [r (lookup-word w)]
+ (let [r (gloss-word w)]
(if r
(l/infof "found word! %s" r)
(l/infof "no entry... %s" w))