wiktlarp

.gitignore

@@ -17,4 +17,5 @@ build/
 enwiktionary-latest-pages-articles.xml.bz2
 raw-wiktexttract-data.jsonl
 *.db
-raw-wiktextract-data.jsonl
+raw-wiktextract-data.jsonlwords.db
+words.db-journal

README.md

@@ -1,3 +1,26 @@
-# wiktionary-tf2
+# wiktlarp-tf2
 
-this is a really stupid script to gloss `[[wikilinked]]` words sent in tf2 chat.
+larp as en.wiktionary.org.
+
+![example](demo.png)
+
+this is a really stupid script to gloss `[[wikilinked]]` words sent in
+tf2 chat. it works by reading TF2's console log file for events, and
+responding over RCON. 
+
+```
+con_logfile console.log
+ip 0.0.0.0
+rcon_password monitor
+net_start
+```
+
+wiktionary entries are discovered via a wiktextract jsonl dump
+converted to sqlite via the rust program in `db`. it is also desirable
+to create an index for the words:
+
+```clj
+(let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})]
+        (with-open [conn (sql/get-connection ds)]
+          (sql/execute! conn ["create index idx on words(word)"])))
+```

db/.gitignore

@@ -0,0 +1,8 @@
+raw-wiktextract-data.jsonl.gz
+target/
+words.db
+words.db-journal
+words.db-wal
+words.db-shm
+words.preview.db
+flamegraph.svg

db/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "db"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+clap = { version = "4.6.5", features = ["derive"] }
+clio = { version = "0.3.5", features = ["clap-parse"] }
+diesel = { version = "2.3.11", features = ["r2d2", "sqlite"] }
+flate2 = { version = "1.1.9", features = ["zlib-rs"] }
+indicatif = { version = "0.18.6", features = ["rayon"] }
+rayon = "1.12.0"
+serde = "1.0.229"
+serde_json = "1.0.151"
+wiktionary-schema = "0.3.0"

db/default.nix

@@ -0,0 +1,9 @@
+{ rustPlatform
+}:
+
+rustPlatform.buildRustPackage (finalAttrs: {
+  pname = "make-db";
+  version = "0.1.0";
+  src = ./.;
+  cargoLock.lockFile = ./Cargo.lock;
+})

db/diesel.toml

@@ -0,0 +1,9 @@
+# For documentation on how to configure this file,
+# see https://diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema.rs"
+custom_type_derives = ["diesel::query_builder::QueryId", "Clone"]
+
+[migrations_directory]
+dir = "migrations"

db/migrations/2026-08-04-180241-0000_create-words/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+drop table words

db/migrations/2026-08-04-180241-0000_create-words/up.sql

@@ -0,0 +1,6 @@
+-- Your SQL goes here
+create table words (
+       id integer not null primary key autoincrement,
+       word varchar not null,
+       info text not null
+)

db/src/db.rs

@@ -0,0 +1,94 @@
+use diesel::prelude::*;
+use diesel::connection::SimpleConnection;
+use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool};
+use std::env;
+use crate::models::*;
+
+#[derive(Debug)]
+pub struct Fuck ();
+
+impl CustomizeConnection<SqliteConnection, diesel::r2d2::Error> for Fuck {
+    fn on_acquire (&self, conn: &mut SqliteConnection) -> Result<
+        (), diesel::r2d2::Error
+    > {
+        conn.batch_execute ("
+          PRAGMA busy_timeout = 15000;
+          PRAGMA journal_mode = WAL;
+          PRAGMA synchronous = NORMAL;
+          PRAGMA wal_autocheckpoint = 1000;
+          PRAGMA wal_checkpoint(TRUNCATE);
+          PRAGMA cache_size=-2000000
+        ").unwrap ();
+        Ok (())
+    }
+
+    fn on_release (&self, _conn: SqliteConnection) {}
+}
+
+pub fn get_pool () -> Pool<ConnectionManager<SqliteConnection>> {
+    let url = env::var ("DATABASE_URL")
+        .expect ("DATABASE_URL must be set");
+    let manager = ConnectionManager::<SqliteConnection>::new (url);
+    Pool::builder ()
+        .connection_customizer (Box::new (Fuck ()))
+        .test_on_check_out (true)
+        .build (manager)
+        .expect ("Could not build connection pool")
+}
+
+pub fn connect () -> SqliteConnection {
+    let database_url = env::var ("DATABASE_URL")
+        .expect ("DATABASE_URL must be set");
+    let mut conn = SqliteConnection::establish (&database_url)
+        .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url));
+    conn.batch_execute ("
+      PRAGMA journal_mode = WAL;          -- better write-concurrency
+      PRAGMA synchronous = NORMAL;        -- fsync only in critical moments
+      PRAGMA wal_autocheckpoint = 1000;   -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased
+      PRAGMA wal_checkpoint(TRUNCATE);    -- free some space by truncating possibly massive WAL files from the last run.
+      PRAGMA busy_timeout = 5000;          -- sleep if the database is busy
+      PRAGMA cache_size=-2000000
+    ").unwrap ();
+    conn
+}
+
+pub fn get_word (w : &str) -> Vec<Word> {
+    let conn = &mut connect ();
+    use crate::schema::words::dsl::*;
+    let results = words
+        .filter (word.eq (w))
+        .limit (5)
+        .select (Word::as_select ())
+        .load (conn)
+        .expect ("error loading words");
+    for wd in results {
+        println! ("{}", wd.word);
+    }
+    todo! ()
+}
+
+pub fn entry_as_word (entry : &serde_json::Value) -> Option<&str> {
+    entry.get ("word")?.as_str ()
+}
+
+pub fn insert_word (
+    conn : &mut SqliteConnection,
+    entry : serde_json::Value,
+) -> Option<()> {
+    use crate::schema::words;
+    let word = entry_as_word (&entry)?;
+    let new_word = NewWord {
+        word,
+        info: &entry.to_string (),
+    };
+    diesel::insert_into (words::table)
+        .values (new_word)
+        .execute (conn).map_or_else (
+            |e| {
+                println! ("error for {}: {}", word, e);
+                None
+            },
+            Some
+        )?;
+    Some (())
+}

db/src/main.rs

@@ -0,0 +1,80 @@
+pub mod models;
+pub mod schema;
+pub mod db;
+
+use diesel::SqliteConnection;
+use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
+use flate2::read::GzDecoder;
+use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::ParallelProgressIterator;
+use serde_json::Value;
+use wiktionary_schema::en::WordData;
+use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
+use clap::Parser;
+use rayon::prelude::*;
+
+/// Build an SQLite database from a Wiktextract JSONLines dump.
+#[derive(Parser, Debug)]
+#[command(name = "make-db", version, about, long_about = None)]
+struct Args {
+    /// Input file
+    #[clap(long, value_parser)]
+    wiktextract_dump : PathBuf,
+
+    /// Output file
+    #[clap(long, value_parser)]
+    database : Option<PathBuf>,
+}
+
+fn open_wiktextract_dump (path : PathBuf) -> Result<
+    BufReader<GzDecoder<File>>, io::Error
+> {
+    Ok (BufReader::new (GzDecoder::new (File::open (path)?)))
+}
+
+#[allow(nonstandard_style)]
+const wiktextract_dump_lines : u64 = 10736399;
+
+fn do_line (
+    conn: &mut PooledConnection<ConnectionManager<SqliteConnection>>,
+    retries : &RwLock<Vec<String>>,
+    line : String
+) {
+    if let Ok (word) = serde_json::from_str::<Value> (&line) {
+        if let None = db::insert_word (conn, word) {
+            retries.write ().unwrap ().push (line);
+        }
+    }
+}
+
+fn main() -> Result<(), String> {
+    let args = Args::parse ();
+    let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
+    let bar = ProgressBar::new (wiktextract_dump_lines);
+    bar.set_style (
+        ProgressStyle::with_template(
+            "{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
+        ) .unwrap()
+    );
+    let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
+        .par_bridge ()
+        .progress_with (bar);
+    let pool = db::get_pool ();
+    let retries = RwLock::new (vec! []);
+
+    lines.for_each (|line| {
+        let conn = &mut pool.clone ().get ().unwrap ();
+        do_line (conn, &retries, line)
+    });
+
+    println! ("do retries now");
+    let conn = &mut pool.clone ().get ().unwrap ();
+    for line in retries.read ().unwrap ().iter () {
+        println! ("retrying");
+        if let Ok (word) = serde_json::from_str::<Value> (&line) {
+            db::insert_word (conn, word);
+        }
+    }
+
+    Ok (())
+}

db/src/models.rs

@@ -0,0 +1,17 @@
+use diesel::prelude::*;
+
+#[derive(Queryable, Selectable)]
+#[diesel(table_name = crate::schema::words)]
+#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
+pub struct Word {
+    pub id : i32,
+    pub word : String,
+    pub info : String,
+}
+
+#[derive(Insertable)]
+#[diesel(table_name = crate::schema::words)]
+pub struct NewWord<'a> {
+    pub word : &'a str,
+    pub info : &'a str,
+}

db/src/schema.rs

@@ -0,0 +1,9 @@
+// @generated automatically by Diesel CLI.
+
+diesel::table! {
+    words (id) {
+        id -> Integer,
+        word -> Text,
+        info -> Text,
+    }
+}

default.nix

@@ -19,9 +19,9 @@ let
   #   sqlite3
   # ];
 in mkCljBin' {
-  name = "wiktionary-tf2";
+  name = "wiktlarp";
   version = "0.1.0";
   projectSrc = lib.cleanSource ./.;
   lockfile = ./deps-lock.json;
-  main-ns = "wiktionary-tf2.main";
+  main-ns = "wiktlarp.main";
 }

deps-lock.json

@@ -1,14 +1,6 @@
 {
   "lock-version": 4,
-  "git-deps": [
-    {
-      "lib": "io.github.anteoas/hawkeye",
-      "url": "https://github.com/anteoas/hawkeye.git",
-      "rev": "5b3f5a99e9edfe482174258a864b577f7a4eeb47",
-      "git-dir": "https/github.com/anteoas/hawkeye",
-      "hash": "sha256-ghKXjf4e3U8rlWCpffPRHqvf367RBs+VnSpRF6tI0xk="
-    }
-  ],
+  "git-deps": [],
   "mvn-deps": [
     {
       "mvn-path": "aleph/aleph/0.4.1/aleph-0.4.1.jar",
@@ -246,14 +238,14 @@
       "hash": "sha256-k7OxHaltUXiIDfjFBT8Yz8eByv8Nnd9LPGRyRKnRws8="
     },
     {
-      "mvn-path": "net/java/dev/jna/jna/5.14.0/jna-5.14.0.jar",
+      "mvn-path": "net/java/dev/jna/jna/3.2.2/jna-3.2.2.jar",
       "mvn-repo": "https://repo1.maven.org/maven2/",
-      "hash": "sha256-NO0eHyf6iWvKUNvE6ZzzcylnzsOHp6DV40hsCWc/6MY="
+      "hash": "sha256-/o8Xb7+sDBzy+M19M2EPp8VI9VFy0C0gLjJwF02p2HU="
     },
     {
-      "mvn-path": "net/java/dev/jna/jna/5.14.0/jna-5.14.0.pom",
+      "mvn-path": "net/java/dev/jna/jna/3.2.2/jna-3.2.2.pom",
       "mvn-repo": "https://repo1.maven.org/maven2/",
-      "hash": "sha256-4E4llRUB3yWtx7Hc22xTNzyUiXuE0+FJISknY+4Hrj0="
+      "hash": "sha256-tIAbOJ/rE2ZJaYehnjKRtxAw2BnAmYRvjkF/ayD6xJQ="
     },
     {
       "mvn-path": "org/clojure/clojure/1.10.3/clojure-1.10.3.jar",

deps.edn

@@ -9,7 +9,5 @@
         clj-rcon/clj-rcon {:mvn/version "0.1.1"}
         hawk/hawk {:mvn/version "0.2.11"}
         ch.qos.logback/logback-classic #:mvn{:version "1.1.3"}
-        io.github.anteoas/hawkeye
-        {:git/sha "5b3f5a99e9edfe482174258a864b577f7a4eeb47"}
         integrant/integrant {:mvn/version "1.0.1"}}
  :paths ["src" "resources"]}

flake.nix

@@ -33,20 +33,25 @@
 
       packages = each-system ({ pkgs, ... }: {
         default = pkgs.callPackage ./default.nix {};
-        make-db = pkgs.callPackage ./make-db {};
+        db = pkgs.callPackage ./db {};
       });
 
       devShells = each-system ({ pkgs, system, ... }: {
 	default = pkgs.mkShell {
-          inputsFrom = [ self.packages.${system}.make-db ];
+          inputsFrom = [ self.packages.${system}.db ];
+          DATABASE_URL = "words.db";
 	  packages = with pkgs; [
 	    zprint
 	    clojure
 	    babashka
             python3
+            (sqlite.override { interactive = true; })
             python314Packages.wiktextract
             sqlite-web
             jq
+            rust-analyzer
+            diesel-cli
+            cargo-flamegraph
 	  ];
 	};
       });

main

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-while read line; do
-    echo $line
-done

make-db/.gitignore

@@ -1 +0,0 @@
-make-db

make-db/Makefile

@@ -1,13 +0,0 @@
-CFLAGS= \
-	-std=c99 \
-	-Wall \
-	-O3 \
-	-Wextra
-LIBS=ncurses sqlite3
-CFLAGS+=$(shell pkg-config --cflags $(LIBS))
-LDFLAGS+=$(shell pkg-config --libs $(LIBS))
-
-SOURCES=$(wildcard *.c)
-EXECUTABLES=$(patsubst %.c,%,$(SOURCES))
-
-all: $(EXECUTABLES)

make-db/default.nix

@@ -1,17 +0,0 @@
-{ stdenv
-, sqlite
-, ncurses
-, pkg-config
-, progressbar
-}:
-
-stdenv.mkDerivation {
-  pname = "make-db";
-  version = "0.0.0";
-  nativeBuildInputs = [
-    sqlite
-    ncurses
-    pkg-config
-    progressbar.dev
-  ];
-}

make-db/make-db.c

@@ -1,45 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <sqlite3.h>
-#include <unistd.h>
-
-const char *word_db_path = "../words.db";
-const char *word_plaintext_path = "../words.gz";
-
-char buf[2048] = "";
-
-char *word = buf;
-
-char *read_entry () {
-  const char *s = fgets (buf, 2048, stdin);
-  if (s == NULL) {
-    return NULL;
-  } else {
-    char *r = strchr (buf, '\t');
-    if (r != NULL) {
-      *r = '\0';
-      return r + 1;
-    } else {
-      return NULL;
-    }
-  }
-}
-
-int main () {
-  if (access (word_db_path, F_OK)) {
-    remove (word_db_path);
-  }
-
-  sqlite3 *db = NULL;
-  if (! sqlite3_open (word_db_path, &db) == SQLITE_OK) {
-    fprintf (stderr, "couldn't open sqlite: %s\n", sqlite3_errmsg (db));
-  } else {
-    fprintf (stderr, "sqlite open }:3\n");
-    char *s = read_entry ();
-    if (s == NULL) {
-      printf ("%s: %s\n", word, s);
-    }
-  }
-
-  return 0;
-}

make-db/progressbar.nix

@@ -1,5 +0,0 @@
-{ stdenv
-, ncurses
-}:
-
-stdenv.mkDerivation {}

src/wiktlarp/main.clj

@@ -1,5 +1,6 @@
-(ns wiktionary-tf2.main
+(ns wiktlarp.main
   (:require [clojure.java.io :as io]
+            [cheshire.core :as json]
             [babashka.fs :as fs]
             [babashka.process :as p]
             [progrock.core :as prog]
@@ -15,24 +16,28 @@
 
 (def ^:dynamic *rcon*)
 
-(def ^:dynamic *dictionary-file*)
+(def ^:dynamic *db*)
 
-(defn parse-plaintext-entry [s]
-  (when-some [[_ word gloss] (re-matches #"([^\t]*)\t(.*)\n?" s)]
-    [word gloss]))
+(defmacro with-conn [[conn] & body]
+  `(let [ds# (sql/get-datasource
+              {:dbtype "sqlite"
+               :dbname "db/words.db"})]
+     (with-open [~conn (sql/get-connection ds#)]
+       (sql/execute! ~conn ["select load_extension (?)"
+                            (System/getenv "SPELLFIX")])
+       ~@body)))
 
-(defn lookup-word [word]
-  (let [r (p/shell {:out :string :continue true}
-                   "zgrep" "-m1"
-                   (format "^%s\t[^[:space:]]" word)
-                   *dictionary-file*)]
-    (when (zero? (:exit r))
-      (->> r
-           :out
-           str/split-lines
-           (map #(-> % parse-plaintext-entry second))
-           (filter #(not (empty? %))) ; remove nil and ""
-           first))))
+(defn query-word [word]
+  (->> (sql/execute! *db* ["select info from words where word = ?"
+                           word])
+       (map #(json/parse-string (:words/info %) keyword))))
+
+(defn gloss-word [word]
+  (->> word
+       query-word
+       (mapcat :senses)
+       (mapcat (some-fn :raw_glosses :glosses))
+       first))
 
 (defn parse-chat-message [x]
   (when-let [[_ _dead? author body]
@@ -54,7 +59,7 @@
 (defn do-definition [s]
   (when-some [w (find-word s)]
     (l/infof "looking up word: %s" w)
-    (let [r (lookup-word w)]
+    (let [r (gloss-word w)]
       (if r
         (l/infof "found word! %s" r)
         (l/infof "no entry... %s" w))
@@ -65,7 +70,7 @@
 (defn rcon-connect [host port password]
   (l/info "attempting rcon connection...")
   (or (try (let [c @(rcon/connect host port password)]
-             @(rcon/exec c "echo \"WIKTIONARY CONNECTED!!!\"")
+             @(rcon/exec c "echo \"wikilarper connected!\"")
              (l/info "connected to rcon!")
              c)
            (catch java.net.ConnectException e
@@ -99,6 +104,8 @@
   {:rcon/connection {:ip "127.0.0.1"
                      :port 27015
                      :password "monitor"}
+   :database/connection {:dbtype "sqlite"
+                         :dbname "db/words.db"}
    :console/watcher
    {:log-file
     (-> (str "~/.local/share/Steam/steamapps/common/Team Fortress 2/"
@@ -107,8 +114,7 @@
         fs/file)
     :handler (ig/ref :wikilinker/handler)}
    :wikilinker/handler {:rcon (ig/ref :rcon/connection)
-                        :dictionary-file "words.gz"}})
-
+                        :db (ig/ref :database/connection)}})
 
 (defmethod ig/init-key :rcon/connection [_ {:keys [ip port password]}]
   (rcon-connect ip port password))
@@ -116,6 +122,12 @@
 (defmethod ig/halt-key! :rcon/connection [_ conn]
   (.close conn))
 
+(defmethod ig/init-key :database/connection [_ dsinfo]
+  (sql/get-datasource dsinfo))
+
+(defmethod ig/halt-key! :database/connection [_ conn]
+  (.close conn))
+
 (defmethod ig/init-key :console/watcher [_ {:keys [log-file handler]}]
   (l/infof "watching file %s" log-file)
   (hawk/watch!
@@ -134,9 +146,9 @@
   (hawk/stop! watcher))
 
 (defmethod ig/init-key :wikilinker/handler
-  [_ {:keys [rcon dictionary-file]}]
+  [_ {:keys [rcon db]}]
   (binding [*rcon* rcon
-            *dictionary-file* dictionary-file]
+            *db* db]
     (bound-fn [log-file] (console-handler log-file))))