diff --git a/db/.gitignore b/db/.gitignore
index 1de5659..543b0b0 100644
--- a/db/.gitignore
+++ b/db/.gitignore
@@ -1 +1,8 @@
-target
\ No newline at end of file
+raw-wiktextract-data.jsonl.gz
+target/
+words.db
+words.db-journal
+words.db-wal
+words.db-shm
+words.preview.db
+flamegraph.svg
\ No newline at end of file
diff --git a/db/Cargo.lock b/db/Cargo.lock
index 8164f44..379c055 100644
--- a/db/Cargo.lock
+++ b/db/Cargo.lock
@@ -168,6 +168,31 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
+
[[package]]
name = "darling"
version = "0.21.3"
@@ -212,6 +237,7 @@ dependencies = [
"diesel",
"flate2",
"indicatif",
+ "rayon",
"serde",
"serde_json",
"wiktionary-schema",
@@ -232,6 +258,7 @@ dependencies = [
"diesel_derives",
"downcast-rs",
"libsqlite3-sys",
+ "r2d2",
"sqlite-wasm-rs",
"time",
]
@@ -405,6 +432,7 @@ checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c"
dependencies = [
"console",
"portable-atomic",
+ "rayon",
"unicode-width",
"unit-prefix",
"web-time",
@@ -466,6 +494,21 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+[[package]]
+name = "lock_api"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+
[[package]]
name = "memchr"
version = "2.8.3"
@@ -500,6 +543,29 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -548,6 +614,46 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+[[package]]
+name = "r2d2"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
+dependencies = [
+ "log",
+ "parking_lot",
+ "scheduled-thread-pool",
+]
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
[[package]]
name = "rsqlite-vfs"
version = "0.1.1"
@@ -586,6 +692,21 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "scheduled-thread-pool"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
+dependencies = [
+ "parking_lot",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
[[package]]
name = "serde"
version = "1.0.229"
@@ -647,6 +768,12 @@ version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+[[package]]
+name = "smallvec"
+version = "1.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+
[[package]]
name = "sqlite-wasm-rs"
version = "0.5.5"
diff --git a/db/Cargo.toml b/db/Cargo.toml
index 442e935..cfe1a13 100644
--- a/db/Cargo.toml
+++ b/db/Cargo.toml
@@ -6,9 +6,10 @@ edition = "2024"
[dependencies]
clap = { version = "4.6.5", features = ["derive"] }
clio = { version = "0.3.5", features = ["clap-parse"] }
-diesel = { version = "2.3.11", features = ["sqlite"] }
+diesel = { version = "2.3.11", features = ["r2d2", "sqlite"] }
flate2 = { version = "1.1.9", features = ["zlib-rs"] }
-indicatif = "0.18.6"
+indicatif = { version = "0.18.6", features = ["rayon"] }
+rayon = "1.12.0"
serde = "1.0.229"
serde_json = "1.0.151"
wiktionary-schema = "0.3.0"
diff --git a/db/flamegraph.svg b/db/flamegraph.svg
new file mode 100644
index 0000000..e5a306a
--- /dev/null
+++ b/db/flamegraph.svg
@@ -0,0 +1,491 @@
+
\ No newline at end of file
diff --git a/db/src/db.rs b/db/src/db.rs
index d59372f..7a7bdf9 100644
--- a/db/src/db.rs
+++ b/db/src/db.rs
@@ -1,12 +1,55 @@
use diesel::prelude::*;
+use diesel::connection::SimpleConnection;
+use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool};
use std::env;
use crate::models::*;
+#[derive(Debug)]
+pub struct Fuck ();
+
+impl CustomizeConnection for Fuck {
+ fn on_acquire (&self, conn: &mut SqliteConnection) -> Result<
+ (), diesel::r2d2::Error
+ > {
+ conn.batch_execute ("
+ PRAGMA busy_timeout = 15000;
+ PRAGMA journal_mode = WAL;
+ PRAGMA synchronous = NORMAL;
+ PRAGMA wal_autocheckpoint = 1000;
+ PRAGMA wal_checkpoint(TRUNCATE);
+ PRAGMA cache_size=-2000000
+ ").unwrap ();
+ Ok (())
+ }
+
+ fn on_release (&self, _conn: SqliteConnection) {}
+}
+
+pub fn get_pool () -> Pool> {
+ let url = env::var ("DATABASE_URL")
+ .expect ("DATABASE_URL must be set");
+ let manager = ConnectionManager::::new (url);
+ Pool::builder ()
+ .connection_customizer (Box::new (Fuck ()))
+ .test_on_check_out (true)
+ .build (manager)
+ .expect ("Could not build connection pool")
+}
+
pub fn connect () -> SqliteConnection {
let database_url = env::var ("DATABASE_URL")
.expect ("DATABASE_URL must be set");
- SqliteConnection::establish (&database_url)
- .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url))
+ let mut conn = SqliteConnection::establish (&database_url)
+ .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url));
+ conn.batch_execute ("
+ PRAGMA journal_mode = WAL; -- better write-concurrency
+ PRAGMA synchronous = NORMAL; -- fsync only in critical moments
+ PRAGMA wal_autocheckpoint = 1000; -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased
+ PRAGMA wal_checkpoint(TRUNCATE); -- free some space by truncating possibly massive WAL files from the last run.
+ PRAGMA busy_timeout = 5000; -- sleep if the database is busy
+ PRAGMA cache_size=-2000000
+ ").unwrap ();
+ conn
}
pub fn get_word (w : &str) -> Vec {
@@ -39,7 +82,6 @@ pub fn insert_word (
};
diesel::insert_into (words::table)
.values (new_word)
- .execute (conn)
- .expect ("error saving new word");
+ .execute (conn).ok ()?;
Some (())
}
diff --git a/db/src/main.rs b/db/src/main.rs
index c490705..18fc983 100644
--- a/db/src/main.rs
+++ b/db/src/main.rs
@@ -4,10 +4,12 @@ pub mod db;
use flate2::read::GzDecoder;
use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::ParallelProgressIterator;
use serde_json::Value;
use wiktionary_schema::en::WordData;
-use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf};
+use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
use clap::Parser;
+use rayon::prelude::*;
/// Build an SQLite database from a Wiktextract JSONLines dump.
#[derive(Parser, Debug)]
@@ -40,18 +42,19 @@ fn main() -> Result<(), String> {
"{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
) .unwrap()
);
- let conn = &mut db::connect ();
+ let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
+ .par_bridge ()
+ .progress_with (bar);
+ let pool = db::get_pool ();
- for i in d.lines () {
+ lines.for_each (|line| {
+ let conn = &mut pool.clone ().get ().unwrap ();
if let Ok (word) = serde_json::from_str:: (
- &i.expect ("entry")
+ &line
) {
- bar.println (db::entry_as_word (&word).map_or ("", |x| x));
db::insert_word (conn, word);
- bar.inc (1);
}
- }
+ });
- bar.finish ();
Ok (())
}
diff --git a/db/words.db-journal b/db/words.db-journal
deleted file mode 100644
index 5faf9f7..0000000
Binary files a/db/words.db-journal and /dev/null differ
diff --git a/flake.nix b/flake.nix
index c774553..6cf4211 100644
--- a/flake.nix
+++ b/flake.nix
@@ -51,6 +51,7 @@
jq
rust-analyzer
diesel-cli
+ cargo-flamegraph
];
};
});
diff --git a/src/wiktionary_tf2/main.clj b/src/wiktionary_tf2/main.clj
index 95c982c..06bf44c 100644
--- a/src/wiktionary_tf2/main.clj
+++ b/src/wiktionary_tf2/main.clj
@@ -1,5 +1,6 @@
(ns wiktionary-tf2.main
(:require [clojure.java.io :as io]
+ [cheshire.core :as json]
[babashka.fs :as fs]
[babashka.process :as p]
[progrock.core :as prog]
@@ -22,16 +23,13 @@
[word gloss]))
(defn lookup-word [word]
- (let [r (p/shell {:out :string :continue true}
- "zgrep" "-m1"
- (format "^%s\t[^[:space:]]" word)
- *dictionary-file*)]
- (when (zero? (:exit r))
- (->> r
- :out
- str/split-lines
- (map #(-> % parse-plaintext-entry second))
- (filter #(not (empty? %))) ; remove nil and ""
+ (let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})]
+ (with-open [conn (sql/get-connection ds)]
+ (->> (sql/execute! conn ["select info from words where word = ?"
+ word])
+ (map #(json/parse-string (:words/info %) keyword))
+ (mapcat :senses)
+ (mapcat :raw_glosses)
first))))
(defn parse-chat-message [x]