excellent

db/.gitignore

@@ -1 +1,8 @@
-target
+raw-wiktextract-data.jsonl.gz
+target/
+words.db
+words.db-journal
+words.db-wal
+words.db-shm
+words.preview.db
+flamegraph.svg

db/Cargo.lock

@@ -168,6 +168,31 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
+
 [[package]]
 name = "darling"
 version = "0.21.3"
@@ -212,6 +237,7 @@ dependencies = [
  "diesel",
  "flate2",
  "indicatif",
+ "rayon",
  "serde",
  "serde_json",
  "wiktionary-schema",
@@ -232,6 +258,7 @@ dependencies = [
  "diesel_derives",
  "downcast-rs",
  "libsqlite3-sys",
+ "r2d2",
  "sqlite-wasm-rs",
  "time",
 ]
@@ -405,6 +432,7 @@ checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c"
 dependencies = [
  "console",
  "portable-atomic",
+ "rayon",
  "unicode-width",
  "unit-prefix",
  "web-time",
@@ -466,6 +494,21 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
 
+[[package]]
+name = "lock_api"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+
 [[package]]
 name = "memchr"
 version = "2.8.3"
@@ -500,6 +543,29 @@ version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.17"
@@ -548,6 +614,46 @@ version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
 
+[[package]]
+name = "r2d2"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
+dependencies = [
+ "log",
+ "parking_lot",
+ "scheduled-thread-pool",
+]
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "rsqlite-vfs"
 version = "0.1.1"
@@ -586,6 +692,21 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
+dependencies = [
+ "parking_lot",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "serde"
 version = "1.0.229"
@@ -647,6 +768,12 @@ version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
+[[package]]
+name = "smallvec"
+version = "1.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+
 [[package]]
 name = "sqlite-wasm-rs"
 version = "0.5.5"

db/Cargo.toml

@@ -6,9 +6,10 @@ edition = "2024"
 [dependencies]
 clap = { version = "4.6.5", features = ["derive"] }
 clio = { version = "0.3.5", features = ["clap-parse"] }
-diesel = { version = "2.3.11", features = ["sqlite"] }
+diesel = { version = "2.3.11", features = ["r2d2", "sqlite"] }
 flate2 = { version = "1.1.9", features = ["zlib-rs"] }
-indicatif = "0.18.6"
+indicatif = { version = "0.18.6", features = ["rayon"] }
+rayon = "1.12.0"
 serde = "1.0.229"
 serde_json = "1.0.151"
 wiktionary-schema = "0.3.0"

db/src/db.rs

@@ -1,12 +1,55 @@
 use diesel::prelude::*;
+use diesel::connection::SimpleConnection;
+use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool};
 use std::env;
 use crate::models::*;
 
+#[derive(Debug)]
+pub struct Fuck ();
+
+impl CustomizeConnection<SqliteConnection, diesel::r2d2::Error> for Fuck {
+    fn on_acquire (&self, conn: &mut SqliteConnection) -> Result<
+        (), diesel::r2d2::Error
+    > {
+        conn.batch_execute ("
+          PRAGMA busy_timeout = 15000;
+          PRAGMA journal_mode = WAL;
+          PRAGMA synchronous = NORMAL;
+          PRAGMA wal_autocheckpoint = 1000;
+          PRAGMA wal_checkpoint(TRUNCATE);
+          PRAGMA cache_size=-2000000
+        ").unwrap ();
+        Ok (())
+    }
+
+    fn on_release (&self, _conn: SqliteConnection) {}
+}
+
+pub fn get_pool () -> Pool<ConnectionManager<SqliteConnection>> {
+    let url = env::var ("DATABASE_URL")
+        .expect ("DATABASE_URL must be set");
+    let manager = ConnectionManager::<SqliteConnection>::new (url);
+    Pool::builder ()
+        .connection_customizer (Box::new (Fuck ()))
+        .test_on_check_out (true)
+        .build (manager)
+        .expect ("Could not build connection pool")
+}
+
 pub fn connect () -> SqliteConnection {
     let database_url = env::var ("DATABASE_URL")
         .expect ("DATABASE_URL must be set");
-    SqliteConnection::establish (&database_url)
+    let mut conn = SqliteConnection::establish (&database_url)
-        .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url))
+        .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url));
+    conn.batch_execute ("
+      PRAGMA journal_mode = WAL;          -- better write-concurrency
+      PRAGMA synchronous = NORMAL;        -- fsync only in critical moments
+      PRAGMA wal_autocheckpoint = 1000;   -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased
+      PRAGMA wal_checkpoint(TRUNCATE);    -- free some space by truncating possibly massive WAL files from the last run.
+      PRAGMA busy_timeout = 5000;          -- sleep if the database is busy
+      PRAGMA cache_size=-2000000
+    ").unwrap ();
+    conn
 }
 
 pub fn get_word (w : &str) -> Vec<Word> {
@@ -39,7 +82,6 @@ pub fn insert_word (
     };
     diesel::insert_into (words::table)
         .values (new_word)
-        .execute (conn)
+        .execute (conn).ok ()?;
-        .expect ("error saving new word");
     Some (())
 }

db/src/main.rs

@@ -4,10 +4,12 @@ pub mod db;
 
 use flate2::read::GzDecoder;
 use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::ParallelProgressIterator;
 use serde_json::Value;
 use wiktionary_schema::en::WordData;
-use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf};
+use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
 use clap::Parser;
+use rayon::prelude::*;
 
 /// Build an SQLite database from a Wiktextract JSONLines dump.
 #[derive(Parser, Debug)]
@@ -40,18 +42,19 @@ fn main() -> Result<(), String> {
             "{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
         ) .unwrap()
     );
-    let conn = &mut db::connect ();
+    let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
+        .par_bridge ()
+        .progress_with (bar);
+    let pool = db::get_pool ();
 
-    for i in d.lines () {
+    lines.for_each (|line| {
+        let conn = &mut pool.clone ().get ().unwrap ();
         if let Ok (word) = serde_json::from_str::<Value> (
-            &i.expect ("entry")
+            &line
         ) {
-            bar.println (db::entry_as_word (&word).map_or ("", |x| x));
             db::insert_word (conn, word);
-            bar.inc (1);
         }
-    }
+    });
 
-    bar.finish ();
     Ok (())
 }

flake.nix

@@ -51,6 +51,7 @@
             jq
             rust-analyzer
             diesel-cli
+            cargo-flamegraph
 	  ];
 	};
       });

src/wiktionary_tf2/main.clj

@@ -1,5 +1,6 @@
 (ns wiktionary-tf2.main
   (:require [clojure.java.io :as io]
+            [cheshire.core :as json]
             [babashka.fs :as fs]
             [babashka.process :as p]
             [progrock.core :as prog]
@@ -22,16 +23,13 @@
     [word gloss]))
 
 (defn lookup-word [word]
-  (let [r (p/shell {:out :string :continue true}
+  (let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})]
-                   "zgrep" "-m1"
+    (with-open [conn (sql/get-connection ds)]
-                   (format "^%s\t[^[:space:]]" word)
+      (->> (sql/execute! conn ["select info from words where word = ?"
-                   *dictionary-file*)]
+                               word])
-    (when (zero? (:exit r))
+           (map #(json/parse-string (:words/info %) keyword))
-      (->> r
+           (mapcat :senses)
-           :out
+           (mapcat :raw_glosses)
-           str/split-lines
-           (map #(-> % parse-plaintext-entry second))
-           (filter #(not (empty? %))) ; remove nil and ""
            first))))
 
 (defn parse-chat-message [x]