excellent

db/src/db.rs

@@ -1,12 +1,55 @@
 use diesel::prelude::*;
+use diesel::connection::SimpleConnection;
+use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool};
 use std::env;
 use crate::models::*;
 
+#[derive(Debug)]
+pub struct Fuck ();
+
+impl CustomizeConnection<SqliteConnection, diesel::r2d2::Error> for Fuck {
+    fn on_acquire (&self, conn: &mut SqliteConnection) -> Result<
+        (), diesel::r2d2::Error
+    > {
+        conn.batch_execute ("
+          PRAGMA busy_timeout = 15000;
+          PRAGMA journal_mode = WAL;
+          PRAGMA synchronous = NORMAL;
+          PRAGMA wal_autocheckpoint = 1000;
+          PRAGMA wal_checkpoint(TRUNCATE);
+          PRAGMA cache_size=-2000000
+        ").unwrap ();
+        Ok (())
+    }
+
+    fn on_release (&self, _conn: SqliteConnection) {}
+}
+
+pub fn get_pool () -> Pool<ConnectionManager<SqliteConnection>> {
+    let url = env::var ("DATABASE_URL")
+        .expect ("DATABASE_URL must be set");
+    let manager = ConnectionManager::<SqliteConnection>::new (url);
+    Pool::builder ()
+        .connection_customizer (Box::new (Fuck ()))
+        .test_on_check_out (true)
+        .build (manager)
+        .expect ("Could not build connection pool")
+}
+
 pub fn connect () -> SqliteConnection {
     let database_url = env::var ("DATABASE_URL")
         .expect ("DATABASE_URL must be set");
-    SqliteConnection::establish (&database_url)
-        .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url))
+    let mut conn = SqliteConnection::establish (&database_url)
+        .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url));
+    conn.batch_execute ("
+      PRAGMA journal_mode = WAL;          -- better write-concurrency
+      PRAGMA synchronous = NORMAL;        -- fsync only in critical moments
+      PRAGMA wal_autocheckpoint = 1000;   -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased
+      PRAGMA wal_checkpoint(TRUNCATE);    -- free some space by truncating possibly massive WAL files from the last run.
+      PRAGMA busy_timeout = 5000;          -- sleep if the database is busy
+      PRAGMA cache_size=-2000000
+    ").unwrap ();
+    conn
 }
 
 pub fn get_word (w : &str) -> Vec<Word> {
@@ -39,7 +82,6 @@ pub fn insert_word (
     };
     diesel::insert_into (words::table)
         .values (new_word)
-        .execute (conn)
-        .expect ("error saving new word");
+        .execute (conn).ok ()?;
     Some (())
 }

db/src/main.rs

@@ -4,10 +4,12 @@ pub mod db;
 
 use flate2::read::GzDecoder;
 use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::ParallelProgressIterator;
 use serde_json::Value;
 use wiktionary_schema::en::WordData;
-use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf};
+use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
 use clap::Parser;
+use rayon::prelude::*;
 
 /// Build an SQLite database from a Wiktextract JSONLines dump.
 #[derive(Parser, Debug)]
@@ -40,18 +42,19 @@ fn main() -> Result<(), String> {
             "{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
         ) .unwrap()
     );
-    let conn = &mut db::connect ();
+    let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
+        .par_bridge ()
+        .progress_with (bar);
+    let pool = db::get_pool ();
 
-    for i in d.lines () {
+    lines.for_each (|line| {
+        let conn = &mut pool.clone ().get ().unwrap ();
         if let Ok (word) = serde_json::from_str::<Value> (
-            &i.expect ("entry")
+            &line
         ) {
-            bar.println (db::entry_as_word (&word).map_or ("", |x| x));
             db::insert_word (conn, word);
-            bar.inc (1);
         }
-    }
+    });
 
-    bar.finish ();
     Ok (())
 }