msyds/wiktlarp-tf2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

excellent

Browse Source
This commit is contained in:
Madeleine Sydney Slaga
2026-08-04 14:03:38 -06:00
parent 05d48c0c3b
commit c4d4bc8735
9 changed files with 695 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/.gitignore
+8 -1
View File
@@ -1 +1,8 @@
target
raw-wiktextract-data.jsonl.gz
target/
words.db
words.db-journal
words.db-wal
words.db-shm
words.preview.db
flamegraph.svg
db/Cargo.lock
Generated
+127
View File
@@ -168,6 +168,31 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
[[package]]
name = "darling"
version = "0.21.3"
@@ -212,6 +237,7 @@ dependencies = [
"diesel",
"flate2",
"indicatif",
"rayon",
"serde",
"serde_json",
"wiktionary-schema",
@@ -232,6 +258,7 @@ dependencies = [
"diesel_derives",
"downcast-rs",
"libsqlite3-sys",
"r2d2",
"sqlite-wasm-rs",
"time",
]
@@ -405,6 +432,7 @@ checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c"
dependencies = [
"console",
"portable-atomic",
"rayon",
"unicode-width",
"unit-prefix",
"web-time",
@@ -466,6 +494,21 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "memchr"
version = "2.8.3"
@@ -500,6 +543,29 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -548,6 +614,46 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "r2d2"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
dependencies = [
"log",
"parking_lot",
"scheduled-thread-pool",
]
[[package]]
name = "rayon"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags",
]
[[package]]
name = "rsqlite-vfs"
version = "0.1.1"
@@ -586,6 +692,21 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scheduled-thread-pool"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
dependencies = [
"parking_lot",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "serde"
version = "1.0.229"
@@ -647,6 +768,12 @@ version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
[[package]]
name = "smallvec"
version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "sqlite-wasm-rs"
version = "0.5.5"
db/Cargo.toml
+3 -2
View File
@@ -6,9 +6,10 @@ edition = "2024"
[dependencies]
clap = { version = "4.6.5", features = ["derive"] }
clio = { version = "0.3.5", features = ["clap-parse"] }
diesel = { version = "2.3.11", features = ["sqlite"] }
diesel = { version = "2.3.11", features = ["r2d2", "sqlite"] }
flate2 = { version = "1.1.9", features = ["zlib-rs"] }
indicatif = "0.18.6"
indicatif = { version = "0.18.6", features = ["rayon"] }
rayon = "1.12.0"
serde = "1.0.229"
serde_json = "1.0.151"
wiktionary-schema = "0.3.0"
db/flamegraph.svg
+491
View File
File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 114 KiB

db/src/db.rs
+46 -4
View File
@@ -1,12 +1,55 @@
use diesel::prelude::*;
use diesel::connection::SimpleConnection;
use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool};
use std::env;
use crate::models::*;
#[derive(Debug)]
pub struct Fuck ();
impl CustomizeConnection<SqliteConnection, diesel::r2d2::Error> for Fuck {
fn on_acquire (&self, conn: &mut SqliteConnection) -> Result<
(), diesel::r2d2::Error
> {
conn.batch_execute ("
PRAGMA busy_timeout = 15000;
PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
PRAGMA wal_autocheckpoint = 1000;
PRAGMA wal_checkpoint(TRUNCATE);
PRAGMA cache_size=-2000000
").unwrap ();
Ok (())
}
fn on_release (&self, _conn: SqliteConnection) {}
}
pub fn get_pool () -> Pool<ConnectionManager<SqliteConnection>> {
let url = env::var ("DATABASE_URL")
.expect ("DATABASE_URL must be set");
let manager = ConnectionManager::<SqliteConnection>::new (url);
Pool::builder ()
.connection_customizer (Box::new (Fuck ()))
.test_on_check_out (true)
.build (manager)
.expect ("Could not build connection pool")
}
pub fn connect () -> SqliteConnection {
let database_url = env::var ("DATABASE_URL")
.expect ("DATABASE_URL must be set");
SqliteConnection::establish (&database_url)
.unwrap_or_else (|_| panic! ("Error connecting to {}", database_url))
let mut conn = SqliteConnection::establish (&database_url)
.unwrap_or_else (|_| panic! ("Error connecting to {}", database_url));
conn.batch_execute ("
PRAGMA journal_mode = WAL; -- better write-concurrency
PRAGMA synchronous = NORMAL; -- fsync only in critical moments
PRAGMA wal_autocheckpoint = 1000; -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased
PRAGMA wal_checkpoint(TRUNCATE); -- free some space by truncating possibly massive WAL files from the last run.
PRAGMA busy_timeout = 5000; -- sleep if the database is busy
PRAGMA cache_size=-2000000
").unwrap ();
conn
}
pub fn get_word (w : &str) -> Vec<Word> {
@@ -39,7 +82,6 @@ pub fn insert_word (
};
diesel::insert_into (words::table)
.values (new_word)
.execute (conn)
.expect ("error saving new word");
.execute (conn).ok ()?;
Some (())
}
db/src/main.rs
+11 -8
View File
@@ -4,10 +4,12 @@ pub mod db;
use flate2::read::GzDecoder;
use indicatif::{ProgressBar, ProgressStyle};
use indicatif::ParallelProgressIterator;
use serde_json::Value;
use wiktionary_schema::en::WordData;
use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf};
use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
use clap::Parser;
use rayon::prelude::*;
/// Build an SQLite database from a Wiktextract JSONLines dump.
#[derive(Parser, Debug)]
@@ -40,18 +42,19 @@ fn main() -> Result<(), String> {
"{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
) .unwrap()
);
let conn = &mut db::connect ();
let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
.par_bridge ()
.progress_with (bar);
let pool = db::get_pool ();
for i in d.lines () {
lines.for_each (|line| {
let conn = &mut pool.clone ().get ().unwrap ();
if let Ok (word) = serde_json::from_str::<Value> (
&i.expect ("entry")
&line
) {
bar.println (db::entry_as_word (&word).map_or ("", |x| x));
db::insert_word (conn, word);
bar.inc (1);
}
}
});
bar.finish ();
Ok (())
}
db/words.db-journal
BIN
View File
Binary file not shown.
flake.nix
+1
View File
@@ -51,6 +51,7 @@
jq
rust-analyzer
diesel-cli
cargo-flamegraph
];
};
});
src/wiktionary_tf2/main.clj
+8 -10
View File
@@ -1,5 +1,6 @@
(ns wiktionary-tf2.main
(:require [clojure.java.io :as io]
[cheshire.core :as json]
[babashka.fs :as fs]
[babashka.process :as p]
[progrock.core :as prog]
@@ -22,16 +23,13 @@
[word gloss]))
(defn lookup-word [word]
(let [r (p/shell {:out :string :continue true}
"zgrep" "-m1"
(format "^%s\t[^[:space:]]" word)
*dictionary-file*)]
(when (zero? (:exit r))
(->> r
:out
str/split-lines
(map #(-> % parse-plaintext-entry second))
(filter #(not (empty? %))) ; remove nil and ""
(let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})]
(with-open [conn (sql/get-connection ds)]
(->> (sql/execute! conn ["select info from words where word = ?"
word])
(map #(json/parse-string (:words/info %) keyword))
(mapcat :senses)
(mapcat :raw_glosses)
first))))
(defn parse-chat-message [x]