From c4d4bc873550a4f13d716cf361d4b359c80e4133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Madeleine=20Sydney=20=C5=9Alaga?= Date: Tue, 4 Aug 2026 14:03:38 -0600 Subject: [PATCH] excellent --- db/.gitignore | 9 +- db/Cargo.lock | 127 ++++++++++ db/Cargo.toml | 5 +- db/flamegraph.svg | 491 ++++++++++++++++++++++++++++++++++++ db/src/db.rs | 50 +++- db/src/main.rs | 19 +- db/words.db-journal | Bin 12824 -> 0 bytes flake.nix | 1 + src/wiktionary_tf2/main.clj | 18 +- 9 files changed, 695 insertions(+), 25 deletions(-) create mode 100644 db/flamegraph.svg delete mode 100644 db/words.db-journal diff --git a/db/.gitignore b/db/.gitignore index 1de5659..543b0b0 100644 --- a/db/.gitignore +++ b/db/.gitignore @@ -1 +1,8 @@ -target \ No newline at end of file +raw-wiktextract-data.jsonl.gz +target/ +words.db +words.db-journal +words.db-wal +words.db-shm +words.preview.db +flamegraph.svg \ No newline at end of file diff --git a/db/Cargo.lock b/db/Cargo.lock index 8164f44..379c055 100644 --- a/db/Cargo.lock +++ b/db/Cargo.lock @@ -168,6 +168,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "darling" version = "0.21.3" @@ -212,6 +237,7 @@ dependencies = [ "diesel", "flate2", "indicatif", + "rayon", "serde", "serde_json", "wiktionary-schema", @@ -232,6 +258,7 @@ dependencies = [ "diesel_derives", "downcast-rs", "libsqlite3-sys", + "r2d2", "sqlite-wasm-rs", "time", ] @@ -405,6 +432,7 @@ checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ "console", "portable-atomic", + "rayon", "unicode-width", "unit-prefix", "web-time", @@ -466,6 +494,21 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "memchr" version = "2.8.3" @@ -500,6 +543,29 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -548,6 +614,46 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "rsqlite-vfs" version = "0.1.1" @@ -586,6 +692,21 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.229" @@ -647,6 +768,12 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + [[package]] name = "sqlite-wasm-rs" version = "0.5.5" diff --git a/db/Cargo.toml b/db/Cargo.toml index 442e935..cfe1a13 100644 --- a/db/Cargo.toml +++ b/db/Cargo.toml @@ -6,9 +6,10 @@ edition = "2024" [dependencies] clap = { version = "4.6.5", features = ["derive"] } clio = { version = "0.3.5", features = ["clap-parse"] } -diesel = { version = "2.3.11", features = ["sqlite"] } +diesel = { version = "2.3.11", features = ["r2d2", "sqlite"] } flate2 = { version = "1.1.9", features = ["zlib-rs"] } -indicatif = "0.18.6" +indicatif = { version = "0.18.6", features = ["rayon"] } +rayon = "1.12.0" serde = "1.0.229" serde_json = "1.0.151" wiktionary-schema = "0.3.0" diff --git a/db/flamegraph.svg b/db/flamegraph.svg new file mode 100644 index 0000000..e5a306a --- /dev/null +++ b/db/flamegraph.svg @@ -0,0 +1,491 @@ +Flame Graph Reset ZoomSearch <alloc::string::String as core::fmt::Write>::write_str (1,932,904 samples, 1.24%)<console::utils::StyledObject<D> as core::fmt::Display>::fmt (501,504 samples, 0.32%)<diesel::sqlite::connection::SqliteConnection as diesel::connection::Connection>::establish (1,202,489 samples, 0.77%)<diesel::sqlite::query_builder::SqliteQueryBuilder as diesel::query_builder::QueryBuilder<diesel::sqlite::backend::Sqlite>>::push_sql (1,045,240 samples, 0.67%)[db] (652,185 samples, 0.42%)std::io::Write::write_all (652,185 samples, 0.42%)<serde_json::read::StrRead as serde_json::read::Read>::parse_str (880,273 samples, 0.57%)[unknown] (1,181,670 samples, 0.76%)serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize (1,181,670 samples, 0.76%)[unknown] (2,563,447 samples, 1.65%)indicatif::state::ProgressState::eta (501,504 samples, 0.32%)<diesel::sqlite::connection::SqliteConnection as diesel::connection::SimpleConnection>::batch_execute (829,243 samples, 0.53%)diesel::sqlite::connection::raw::RawConnection::exec (829,243 samples, 0.53%)sqlite3_prepare_v2 (829,243 samples, 0.53%)<indicatif::rayon::ProgressFolder<C> as rayon::iter::plumbing::Folder<T>>::consume (1,330,747 samples, 0.85%)db::db::connect (1,330,747 samples, 0.85%)core::result::unwrap_failed (501,504 samples, 0.32%)core::panicking::panic_fmt (501,504 samples, 0.32%)__rustc::rust_begin_unwind (501,504 samples, 0.32%)std::sys::backtrace::__rust_end_short_backtrace (501,504 samples, 0.32%)std::panicking::panic_handler::{{closure}} (501,504 samples, 0.32%)std::panicking::panic_with_hook (501,504 samples, 0.32%)__rustc::rust_panic (501,504 samples, 0.32%)_Unwind_RaiseException (501,504 samples, 0.32%)_Unwind_RaiseException_Phase2 (501,504 samples, 0.32%)rust_eh_personality (501,504 samples, 0.32%)<&rayon::iter::par_bridge::IterParallelProducer<Iter> as rayon::iter::plumbing::UnindexedProducer>::fold_with (2,010,726 samples, 1.29%)<std::io::Lines<B> as core::iter::traits::iterator::Iterator>::next (679,979 samples, 0.44%)std::io::append_to_string (679,979 samples, 0.44%)<flate2::gz::bufread::GzDecoder<R> as std::io::Read>::read (679,979 samples, 0.44%)flate2::zio::read (679,979 samples, 0.44%)<flate2::mem::Decompress as flate2::zio::Ops>::run (679,979 samples, 0.44%)zlib_rs::stable::Inflate::decompress (679,979 samples, 0.44%)zlib_rs::inflate::inflate (679,979 samples, 0.44%)rayon::iter::plumbing::bridge_unindexed_producer_consumer (2,512,230 samples, 1.61%)rayon_core::registry::in_worker (501,504 samples, 0.32%)rayon_core::join::join_context::{{closure}} (501,504 samples, 0.32%)rayon::iter::plumbing::bridge_unindexed_producer_consumer (501,504 samples, 0.32%)<&rayon::iter::par_bridge::IterParallelProducer<Iter> as rayon::iter::plumbing::UnindexedProducer>::fold_with (501,504 samples, 0.32%)<indicatif::rayon::ProgressFolder<C> as rayon::iter::plumbing::Folder<T>>::consume (501,504 samples, 0.32%)db::db::connect (501,504 samples, 0.32%)<diesel::sqlite::connection::SqliteConnection as diesel::connection::SimpleConnection>::batch_execute (501,504 samples, 0.32%)diesel::sqlite::connection::raw::RawConnection::exec (501,504 samples, 0.32%)sqlite3_exec (501,504 samples, 0.32%)sqlite3_prepare_v2 (501,504 samples, 0.32%)sqlite3LockAndPrepare (501,504 samples, 0.32%)sqlite3Prepare (501,504 samples, 0.32%)sqlite3RunParser (501,504 samples, 0.32%)yy_reduce.isra.0 (501,504 samples, 0.32%)sqlite3Pragma (501,504 samples, 0.32%)sqlite3ReadSchema (501,504 samples, 0.32%)sqlite3Init (501,504 samples, 0.32%)sqlite3InitOne (501,504 samples, 0.32%)btreeBeginTrans (501,504 samples, 0.32%)btreeGetPage (501,504 samples, 0.32%)getPageNormal (501,504 samples, 0.32%)pcache1FetchStage2 (501,504 samples, 0.32%)__GI___pthread_mutex_unlock_usercnt (869,878 samples, 0.56%)pcache1Destroy (1,708,933 samples, 1.10%)pcache1EnforceMaxPage (1,708,933 samples, 1.10%)sqlite3_free (1,708,933 samples, 1.10%)_int_free_chunk (1,708,933 samples, 1.10%)_int_free_maybe_consolidate.part.0 (1,708,933 samples, 1.10%)malloc_consolidate (1,708,933 samples, 1.10%)malloc_consolidate (827,944 samples, 0.53%)pcache1Truncate (2,839,838 samples, 1.82%)p..pcache1TruncateUnsafe (2,839,838 samples, 1.82%)p..sqlite3_free (2,130,706 samples, 1.37%)_int_free_chunk (2,130,706 samples, 1.37%)_int_free_maybe_consolidate.part.0 (2,130,706 samples, 1.37%)unlink_chunk.isra.0 (1,302,762 samples, 0.84%)sqlite3WalCheckpoint (1,202,432 samples, 0.77%)fdatasync (1,202,432 samples, 0.77%)__munmap (1,554,916 samples, 1.00%)sqlite3_free (1,498,544 samples, 0.96%)pthread_mutex_lock@@GLIBC_2.2.5 (1,498,544 samples, 0.96%)sqlite3PagerClose.isra.0 (9,662,196 samples, 6.21%)sqlite3P..sqlite3WalClose (5,113,425 samples, 3.28%)sql..unixShmUnmap (3,910,993 samples, 2.51%)un..unixShmPurge (3,910,993 samples, 2.51%)un..unixGetpagesize (857,533 samples, 0.55%)__sysconf (857,533 samples, 0.55%)<diesel::sqlite::connection::raw::RawConnection as core::ops::drop::Drop>::drop (12,562,857 samples, 8.07%)<diesel::sq..sqlite3Close (11,644,637 samples, 7.48%)sqlite3Clo..sqlite3LeaveMutexAndCloseZombie (11,644,637 samples, 7.48%)sqlite3Lea..sqlite3BtreeClose.isra.0 (10,774,759 samples, 6.92%)sqlite3Bt..sqlite3SchemaClear (1,112,563 samples, 0.71%)deleteTable (1,112,563 samples, 0.71%)sqlite3_free (1,112,563 samples, 0.71%)_int_free_chunk (1,112,563 samples, 0.71%)sqlite3_free (841,037 samples, 0.54%)cfree@GLIBC_2.2.5 (841,037 samples, 0.54%)core::ptr::drop_in_place<diesel::sqlite::connection::SqliteConnection> (14,481,464 samples, 9.30%)core::ptr::dr..<hashbrown::raw::RawTable<T,A> as core::ops::drop::Drop>::drop (1,918,607 samples, 1.23%)<diesel::sqlite::connection::stmt::Statement as core::ops::drop::Drop>::drop (1,918,607 samples, 1.23%)sqlite3_finalize (1,918,607 samples, 1.23%)sqlite3VdbeDelete (1,918,607 samples, 1.23%)vdbeFreeOpArray (1,077,570 samples, 0.69%)sqlite3_free (1,077,570 samples, 0.69%)_int_free_chunk (1,077,570 samples, 0.69%)diesel::connection::instrumentation::get_default_instrumentation (1,335,049 samples, 0.86%)setupLookaside (1,026,983 samples, 0.66%)sqlite3Malloc (1,026,983 samples, 0.66%)sqlite3MemMalloc (1,026,983 samples, 0.66%)__libc_malloc2 (1,026,983 samples, 0.66%)_int_malloc (1,026,983 samples, 0.66%)sqlite3BtreeOpen (990,467 samples, 0.64%)unixOpen (990,467 samples, 0.64%)sqlite3_create_module (1,042,935 samples, 0.67%)sqlite3FindFunction (965,327 samples, 0.62%)__strlen_avx2 (965,327 samples, 0.62%)sqlite3Fts3Init (4,200,793 samples, 2.70%)sq..sqlite3_overload_function (3,157,858 samples, 2.03%)s..sqlite3_mprintf (870,408 samples, 0.56%)sqlite3_vmprintf (870,408 samples, 0.56%)strAccumFinishRealloc (870,408 samples, 0.56%)sqlite3Malloc (870,408 samples, 0.56%)sqlite3MemMalloc (870,408 samples, 0.56%)__libc_malloc2 (870,408 samples, 0.56%)fts5CreateAux (1,231,960 samples, 0.79%)__strlen_avx2 (1,231,960 samples, 0.79%)sqlite3Fts5Init (2,839,973 samples, 1.82%)s..sqlite3_create_function (1,608,013 samples, 1.03%)createFunctionApi (1,608,013 samples, 1.03%)sqlite3CreateFunc (1,608,013 samples, 1.03%)sqlite3FindFunction (501,504 samples, 0.32%)sqlite3HashInsert (501,504 samples, 0.32%)sqlite3DbMallocZero (1,213,206 samples, 0.78%)dbMallocRawFinish (1,213,206 samples, 0.78%)sqlite3Malloc (1,213,206 samples, 0.78%)sqlite3MemMalloc (1,213,206 samples, 0.78%)__libc_malloc2 (1,213,206 samples, 0.78%)sqlite3RtreeInit (3,719,974 samples, 2.39%)sq..sqlite3_create_function (3,719,974 samples, 2.39%)sq..createFunctionApi (3,719,974 samples, 2.39%)cr..sqlite3CreateFunc (3,719,974 samples, 2.39%)sq..sqlite3FindFunction (3,719,974 samples, 2.39%)sq..sqlite3HashInsert (1,025,393 samples, 0.66%)diesel::sqlite::connection::raw::RawConnection::establish (13,401,569 samples, 8.61%)diesel::sqli..openDatabase (13,401,569 samples, 8.61%)openDatabasesqlite3_wal_autocheckpoint (623,379 samples, 0.40%)sqlite3_wal_hook (623,379 samples, 0.40%)<&str as alloc::ffi::c_str::CString::new::SpecNewImpl>::spec_new_impl (843,610 samples, 0.54%)__memmove_avx_unaligned_erms (843,610 samples, 0.54%)<diesel::sqlite::connection::SqliteConnection as diesel::connection::Connection>::establish (16,225,201 samples, 10.42%)<diesel::sqlite..diesel::sqlite::connection::raw::RawConnection::register_sql_function (1,488,583 samples, 0.96%)sqlite3_create_function_v2 (644,973 samples, 0.41%)createFunctionApi (644,973 samples, 0.41%)sqlite3CreateFunc (644,973 samples, 0.41%)sqlite3FindFunction (644,973 samples, 0.41%)sqlite3HashInsert (644,973 samples, 0.41%)sqlite3Malloc (644,973 samples, 0.41%)pthread_mutex_lock@@GLIBC_2.2.5 (644,973 samples, 0.41%)sqlite3VdbeDelete (897,945 samples, 0.58%)vdbeFreeOpArray (897,945 samples, 0.58%)freeP4 (897,945 samples, 0.58%)sqlite3GetToken (1,364,872 samples, 0.88%)sqlite3FinishCoding (706,057 samples, 0.45%)sqlite3VdbeMakeReady (706,057 samples, 0.45%)resolveP2Values (706,057 samples, 0.45%)pagerPagecount (932,498 samples, 0.60%)unixFileSize (932,498 samples, 0.60%)__fstat64 (932,498 samples, 0.60%)pager_wait_on_lock (792,376 samples, 0.51%)unixLock (792,376 samples, 0.51%)__libc_fcntl64 (792,376 samples, 0.51%)__fcntl64_nocancel_adjusted (792,376 samples, 0.51%)btreeBeginTrans (3,166,382 samples, 2.03%)b..sqlite3PagerSharedLock (3,166,382 samples, 2.03%)s..walTryBeginRead (501,504 samples, 0.32%)walIndexReadHdr (501,504 samples, 0.32%)walIndexPageRealloc (501,504 samples, 0.32%)sqlite3Malloc (501,504 samples, 0.32%)sqlite3AddColumn (808,404 samples, 0.52%)sqlite3InitCallback (2,528,650 samples, 1.62%)sqlite3Prepare (2,528,650 samples, 1.62%)sqlite3RunParser (2,528,650 samples, 1.62%)yy_reduce.isra.0 (2,528,650 samples, 1.62%)sqlite3StartTable (1,720,246 samples, 1.10%)sqlite3DbStrDup (853,088 samples, 0.55%)sqlite3DbMallocRawNN (853,088 samples, 0.55%)sqlite3GetToken (2,020,668 samples, 1.30%)sqlite3AddPrimaryKey (1,309,420 samples, 0.84%)sqlite3CreateIndex (1,309,420 samples, 0.84%)sqlite3MPrintf (853,088 samples, 0.55%)sqlite3VMPrintf (853,088 samples, 0.55%)strAccumFinishRealloc (853,088 samples, 0.55%)dbMallocRawFinish (853,088 samples, 0.55%)sqlite3Malloc (853,088 samples, 0.55%)sqlite3MemMalloc (853,088 samples, 0.55%)malloc (853,088 samples, 0.55%)sqlite3EndTable (881,828 samples, 0.57%)sqlite3ShadowTableName (881,828 samples, 0.57%)sqlite3FindTable (881,828 samples, 0.57%)findElementWithHash (881,828 samples, 0.57%)sqlite3InitCallback (6,437,347 samples, 4.13%)sqlit..sqlite3Prepare (6,437,347 samples, 4.13%)sqlit..sqlite3RunParser (6,437,347 samples, 4.13%)sqlit..yy_reduce.isra.0 (4,416,679 samples, 2.84%)yy..sqlite3StartTable (2,225,431 samples, 1.43%)sqlite3CheckObjectName (2,225,431 samples, 1.43%)sqlite3_stricmp (2,225,431 samples, 1.43%)sqlite3_column_text (940,004 samples, 0.60%)valueToText (940,004 samples, 0.60%)sqlite3Select (4,223,649 samples, 2.71%)sq..sqlite3GenerateColumnNames (1,133,298 samples, 0.73%)sqlite3VdbeMemSetStr (1,133,298 samples, 0.73%)__memmove_avx_unaligned_erms (1,133,298 samples, 0.73%)sqlite3_prepare_v2 (7,667,824 samples, 4.93%)sqlite..sqlite3LockAndPrepare (7,667,824 samples, 4.93%)sqlite..sqlite3Prepare (7,667,824 samples, 4.93%)sqlite..sqlite3RunParser (7,667,824 samples, 4.93%)sqlite..yy_reduce.isra.0 (5,266,584 samples, 3.38%)yy_..sqlite3SrcListAppendFromTerm (1,042,935 samples, 0.67%)sqlite3BtreeNext.isra.0 (1,058,960 samples, 0.68%)sqlite3BtreeCommitPhaseTwo (1,322,123 samples, 0.85%)pager_unlock (1,322,123 samples, 0.85%)sqlite3WalEndReadTransaction.part.0 (1,322,123 samples, 0.85%)unixShmLock (1,322,123 samples, 0.85%)sqlite3_prepare_v2 (27,573,568 samples, 17.71%)sqlite3_prepare_v2sqlite3LockAndPrepare (27,573,568 samples, 17.71%)sqlite3LockAndPreparesqlite3Prepare (27,573,568 samples, 17.71%)sqlite3Preparesqlite3RunParser (27,573,568 samples, 17.71%)sqlite3RunParseryy_reduce.isra.0 (24,914,358 samples, 16.00%)yy_reduce.isra.0sqlite3Pragma (24,208,301 samples, 15.55%)sqlite3Pragmasqlite3ReadSchema (24,208,301 samples, 15.55%)sqlite3ReadSchemasqlite3Init (24,208,301 samples, 15.55%)sqlite3Initsqlite3InitOne (24,208,301 samples, 15.55%)sqlite3InitOnesqlite3_exec (18,513,269 samples, 11.89%)sqlite3_execsqlite3_step (3,468,094 samples, 2.23%)s..sqlite3VdbeExec (3,468,094 samples, 2.23%)s..sqlite3VdbeHalt (2,409,134 samples, 1.55%)sqlite3VdbeFreeCursorNN (1,087,011 samples, 0.70%)db::db::connect (45,353,189 samples, 29.13%)db::db::connect<diesel::sqlite::connection::SqliteConnection as diesel::connection::SimpleConnection>::batch_execute (29,127,988 samples, 18.71%)<diesel::sqlite::connection::..diesel::sqlite::connection::raw::RawConnection::exec (29,127,988 samples, 18.71%)diesel::sqlite::connection::r..sqlite3_exec (29,127,988 samples, 18.71%)sqlite3_execsqlite3_step (656,475 samples, 0.42%)sqlite3VdbeExec (656,475 samples, 0.42%)sqlite3Checkpoint (656,475 samples, 0.42%)sqlite3WalCheckpoint (656,475 samples, 0.42%)cfree@GLIBC_2.2.5 (1,038,759 samples, 0.67%)_int_free_chunk (5,204,256 samples, 3.34%)_in..alloc::collections::btree::map::IntoIter<K,V,A>::dying_next (1,514,325 samples, 0.97%)_int_free_chunk (448,032 samples, 0.29%)cfree@GLIBC_2.2.5 (1,091,913 samples, 0.70%)<alloc::collections::btree::map::BTreeMap<K,V,A> as core::ops::drop::Drop>::drop (10,933,067 samples, 7.02%)<alloc::c..core::ptr::drop_in_place<serde_json::value::Value> (9,021,329 samples, 5.79%)core::p..<alloc::vec::Vec<T,A> as core::ops::drop::Drop>::drop (9,021,329 samples, 5.79%)<alloc:..core::ptr::drop_in_place<alloc::collections::btree::map::IntoIter<alloc::string::String,serde_json::value::Value>> (9,021,329 samples, 5.79%)core::p..core::ptr::drop_in_place<serde_json::value::Value> (1,210,835 samples, 0.78%)<alloc::vec::Vec<T,A> as core::ops::drop::Drop>::drop (1,210,835 samples, 0.78%)core::ptr::drop_in_place<alloc::collections::btree::map::IntoIter<alloc::string::String,serde_json::value::Value>> (1,210,835 samples, 0.78%)alloc::collections::btree::map::IntoIter<K,V,A>::dying_next (1,210,835 samples, 0.78%)allocateCursor (1,182,090 samples, 0.76%)sqlite3VdbeFreeCursorNN (1,182,090 samples, 0.76%)btreeLast (2,209,826 samples, 1.42%)moveToRoot (2,209,826 samples, 1.42%)getAndInitPage (2,209,826 samples, 1.42%)getPageNormal (2,209,826 samples, 1.42%)pcache1FetchStage2 (2,209,826 samples, 1.42%)sqlite3BlobCompare (1,554,916 samples, 1.00%)sqlite3BtreeInsert (1,093,591 samples, 0.70%)__memmove_avx_unaligned_erms (1,093,591 samples, 0.70%)unixSync (1,083,125 samples, 0.70%)robust_close (1,083,125 samples, 0.70%)__close (1,083,125 samples, 0.70%)__memset_avx2_unaligned_erms (1,468,681 samples, 0.94%)walIndexAppend (2,261,057 samples, 1.45%)memset@plt (792,376 samples, 0.51%)pagerWalFrames (4,287,746 samples, 2.75%)pa..walWriteOneFrame.isra.0 (943,564 samples, 0.61%)walEncodeFrame (943,564 samples, 0.61%)walChecksumBytes (943,564 samples, 0.61%)<diesel::sqlite::connection::SqliteConnection as diesel::connection::Connection>::execute_returning_count (14,927,179 samples, 9.59%)<diesel::sqlit..diesel::sqlite::connection::stmt::StatementUse::run (14,927,179 samples, 9.59%)diesel::sqlite..sqlite3_step (14,927,179 samples, 9.59%)sqlite3_stepsqlite3VdbeExec (13,744,389 samples, 8.83%)sqlite3VdbeE..sqlite3VdbeHalt (4,932,719 samples, 3.17%)sql..sqlite3BtreeCommitPhaseOne.part.0 (4,932,719 samples, 3.17%)sql..sqlite3PagerCommitPhaseOne.part.0 (4,932,719 samples, 3.17%)sql..walWriteOneFrame.isra.0 (644,973 samples, 0.41%)serde_core::ser::SerializeMap::serialize_entry (557,825 samples, 0.36%)serde_core::ser::Serializer::collect_seq (854,934 samples, 0.55%)serde_json::value::ser::<impl serde_core::ser::Serialize for serde_json::value::Value>::serialize (854,934 samples, 0.55%)serde_json::ser::format_escaped_str (854,934 samples, 0.55%)<alloc::string::String as core::fmt::Write>::write_str (1,010,853 samples, 0.65%)alloc::raw_vec::RawVecInner<A>::reserve::do_reserve_and_handle (1,010,853 samples, 0.65%)alloc::raw_vec::RawVecInner<A>::finish_grow (1,010,853 samples, 0.65%)realloc (1,010,853 samples, 0.65%)_int_realloc (1,010,853 samples, 0.65%)__memmove_avx_unaligned_erms (1,010,853 samples, 0.65%)serde_core::ser::Serializer::collect_seq (4,577,800 samples, 2.94%)se..serde_json::value::ser::<impl serde_core::ser::Serialize for serde_json::value::Value>::serialize (4,577,800 samples, 2.94%)se..serde_json::ser::format_escaped_str (3,722,866 samples, 2.39%)se..std::io::Write::write_all (2,771,364 samples, 1.78%)s..<core::fmt::Formatter as core::fmt::Write>::write_str (1,760,511 samples, 1.13%)<alloc::string::String as core::fmt::Write>::write_str (2,021,949 samples, 1.30%)__memmove_avx_unaligned_erms (2,021,949 samples, 1.30%)serde_json::ser::format_escaped_str (4,138,723 samples, 2.66%)se..std::io::Write::write_all (2,960,004 samples, 1.90%)s..<core::fmt::Formatter as core::fmt::Write>::write_str (938,055 samples, 0.60%)serde_json::ser::format_escaped_str (1,613,455 samples, 1.04%)serde_json::value::ser::<impl serde_core::ser::Serialize for serde_json::value::Value>::serialize (4,313,493 samples, 2.77%)se..serde_json::value::ser::<impl serde_core::ser::Serialize for serde_json::value::Value>::serialize (2,700,038 samples, 1.73%)serde_json::ser::format_escaped_str (2,700,038 samples, 1.73%)std::io::Write::write_all (1,038,759 samples, 0.67%)<alloc::string::String as core::fmt::Write>::write_str (1,038,759 samples, 0.67%)__memmove_avx_unaligned_erms (1,038,759 samples, 0.67%)db::db::insert_word (41,233,075 samples, 26.48%)db::db::insert_word<serde_json::value::Value as core::fmt::Display>::fmt (15,372,829 samples, 9.87%)<serde_json::v..serde_core::ser::Serializer::collect_seq (14,815,004 samples, 9.52%)serde_core::se..serde_json::value::ser::<impl serde_core::ser::Serialize for serde_json::value::Value>::serialize (14,815,004 samples, 9.52%)serde_json::va..std::io::Write::write_all (1,784,988 samples, 1.15%)<alloc::string::String as core::fmt::Write>::write_str (1,784,988 samples, 1.15%)__memmove_avx_unaligned_erms (1,784,988 samples, 1.15%)indicatif::progress_bar::ProgressBar::tick_inner (2,889,432 samples, 1.86%)i..indicatif::state::BarState::update_estimate_and_draw (2,889,432 samples, 1.86%)i..pow@@GLIBC_2.29 (2,889,432 samples, 1.86%)p..__ieee754_pow_fma (1,539,727 samples, 0.99%)alloc::raw_vec::RawVec<T,A>::grow_one (889,215 samples, 0.57%)alloc::raw_vec::RawVecInner<A>::finish_grow (889,215 samples, 0.57%)__libc_malloc2 (889,215 samples, 0.57%)<serde_json::de::MapAccess<R> as serde_core::de::MapAccess>::next_key_seed::has_next_key (786,492 samples, 0.51%)<serde_json::de::MapAccess<R> as serde_core::de::MapAccess>::next_key_seed (1,394,303 samples, 0.90%)__libc_malloc2 (607,811 samples, 0.39%)_int_malloc (607,811 samples, 0.39%)alloc::collections::btree::map::BTreeMap<K,V,A>::insert (2,893,469 samples, 1.86%)a..alloc::collections::btree::map::entry::VacantEntry<K,V,A>::insert_entry (2,893,469 samples, 1.86%)a..alloc::collections::btree::node::Handle<alloc::collections::btree::node::NodeRef<alloc::collections::btree::node::marker::Mut,K,V,alloc::collections::btree::node::marker::Leaf>,alloc::collections::btree::node::marker::Edge>::insert_recursing (2,893,469 samples, 1.86%)a..__memmove_avx_unaligned_erms (1,713,388 samples, 1.10%)<serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize::ValueVisitor as serde_core::de::Visitor>::visit_map (1,417,641 samples, 0.91%)alloc::collections::btree::map::BTreeMap<K,V,A>::insert (1,417,641 samples, 0.91%)alloc::collections::btree::map::entry::VacantEntry<K,V,A>::insert_entry (1,417,641 samples, 0.91%)alloc::collections::btree::node::Handle<alloc::collections::btree::node::NodeRef<alloc::collections::btree::node::marker::Mut,K,V,alloc::collections::btree::node::marker::Leaf>,alloc::collections::btree::node::marker::Edge>::insert_recursing (1,417,641 samples, 0.91%)__libc_malloc2 (1,886,098 samples, 1.21%)_int_malloc (607,811 samples, 0.39%)alloc::raw_vec::RawVec<T,A>::grow_one (1,301,901 samples, 0.84%)alloc::raw_vec::RawVecInner<A>::finish_grow (1,301,901 samples, 0.84%)realloc (1,301,901 samples, 0.84%)_int_realloc (1,301,901 samples, 0.84%)_int_malloc (1,301,901 samples, 0.84%)malloc (1,652,383 samples, 1.06%)<serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize::ValueVisitor as serde_core::de::Visitor>::visit_map (12,392,671 samples, 7.96%)<serde_json..serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize (7,288,422 samples, 4.68%)serde..serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize (1,030,399 samples, 0.66%)alloc::raw_vec::RawVec<T,A>::grow_one (1,030,399 samples, 0.66%)alloc::raw_vec::RawVecInner<A>::finish_grow (1,030,399 samples, 0.66%)__libc_malloc2 (1,030,399 samples, 0.66%)<indicatif::rayon::ProgressFolder<C> as rayon::iter::plumbing::Folder<T>>::consume (119,761,498 samples, 76.92%)<indicatif::rayon::ProgressFolder<C> as rayon::iter::plumbing::Folder<T>>::consumeserde_json::de::from_trait (15,804,338 samples, 10.15%)serde_json::de:..serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize (15,804,338 samples, 10.15%)serde_json::val..<serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize::ValueVisitor as serde_core::de::Visitor>::visit_map (15,804,338 samples, 10.15%)<serde_json::va..serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize (15,804,338 samples, 10.15%)serde_json::val..serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize (12,894,175 samples, 8.28%)serde_json::..__libc_malloc2 (501,504 samples, 0.32%)_int_malloc (501,504 samples, 0.32%)zlib_rs::inflate::inflate_fast_help_avx2 (3,484,618 samples, 2.24%)z..<flate2::gz::bufread::GzDecoder<R> as std::io::Read>::read (4,328,228 samples, 2.78%)<f..flate2::zio::read (4,328,228 samples, 2.78%)fl..<flate2::mem::Decompress as flate2::zio::Ops>::run (4,328,228 samples, 2.78%)<f..zlib_rs::stable::Inflate::decompress (4,328,228 samples, 2.78%)zl..zlib_rs::inflate::inflate (4,328,228 samples, 2.78%)zl..zlib_rs::inflate::inftrees::inflate_table (843,610 samples, 0.54%)__GI___clone3 (127,564,696 samples, 81.94%)__GI___clone3start_thread (127,564,696 samples, 81.94%)start_threadstd::sys::thread::unix::Thread::new::thread_start (127,564,696 samples, 81.94%)std::sys::thread::unix::Thread::new::thread_startcore::ops::function::FnOnce::call_once{{vtable.shim}} (127,564,696 samples, 81.94%)core::ops::function::FnOnce::call_once{{vtable.shim}}std::sys::backtrace::__rust_begin_short_backtrace (127,564,696 samples, 81.94%)std::sys::backtrace::__rust_begin_short_backtracerayon_core::registry::ThreadBuilder::run (127,564,696 samples, 81.94%)rayon_core::registry::ThreadBuilder::runrayon_core::registry::WorkerThread::wait_until_cold (127,564,696 samples, 81.94%)rayon_core::registry::WorkerThread::wait_until_cold<rayon_core::job::StackJob<L,F,R> as rayon_core::job::Job>::execute (127,564,696 samples, 81.94%)<rayon_core::job::StackJob<L,F,R> as rayon_core::job::Job>::executerayon_core::join::join_context::{{closure}} (125,052,466 samples, 80.32%)rayon_core::join::join_context::{{closure}}rayon::iter::plumbing::bridge_unindexed_producer_consumer (125,052,466 samples, 80.32%)rayon::iter::plumbing::bridge_unindexed_producer_consumerrayon_core::registry::in_worker (125,052,466 samples, 80.32%)rayon_core::registry::in_workerrayon_core::join::join_context::{{closure}} (125,052,466 samples, 80.32%)rayon_core::join::join_context::{{closure}}rayon::iter::plumbing::bridge_unindexed_producer_consumer (125,052,466 samples, 80.32%)rayon::iter::plumbing::bridge_unindexed_producer_consumerrayon_core::registry::in_worker (125,052,466 samples, 80.32%)rayon_core::registry::in_workerrayon_core::join::join_context::{{closure}} (125,052,466 samples, 80.32%)rayon_core::join::join_context::{{closure}}rayon::iter::plumbing::bridge_unindexed_producer_consumer (125,052,466 samples, 80.32%)rayon::iter::plumbing::bridge_unindexed_producer_consumerrayon_core::registry::in_worker (125,052,466 samples, 80.32%)rayon_core::registry::in_workerrayon_core::join::join_context::{{closure}} (125,052,466 samples, 80.32%)rayon_core::join::join_context::{{closure}}rayon::iter::plumbing::bridge_unindexed_producer_consumer (125,052,466 samples, 80.32%)rayon::iter::plumbing::bridge_unindexed_producer_consumerrayon_core::registry::in_worker (125,052,466 samples, 80.32%)rayon_core::registry::in_workerrayon_core::join::join_context::{{closure}} (125,052,466 samples, 80.32%)rayon_core::join::join_context::{{closure}}rayon::iter::plumbing::bridge_unindexed_producer_consumer (125,052,466 samples, 80.32%)rayon::iter::plumbing::bridge_unindexed_producer_consumer<&rayon::iter::par_bridge::IterParallelProducer<Iter> as rayon::iter::plumbing::UnindexedProducer>::fold_with (125,052,466 samples, 80.32%)<&rayon::iter::par_bridge::IterParallelProducer<Iter> as rayon::iter::plumbing::UnindexedProducer>::fold_with<std::io::Lines<B> as core::iter::traits::iterator::Iterator>::next (5,290,968 samples, 3.40%)<st..std::io::append_to_string (5,290,968 samples, 3.40%)std..alloc::raw_vec::RawVecInner<A>::reserve::do_reserve_and_handle (962,740 samples, 0.62%)alloc::raw_vec::RawVecInner<A>::finish_grow (962,740 samples, 0.62%)__libc_malloc2 (962,740 samples, 0.62%)_int_malloc (962,740 samples, 0.62%)__syscall_cancel_arch_end (2,303,068 samples, 1.48%)_start (501,504 samples, 0.32%)_dl_start (501,504 samples, 0.32%)_dl_sysdep_start (501,504 samples, 0.32%)dl_main (501,504 samples, 0.32%)_dl_relocate_object (501,504 samples, 0.32%)_dl_relocate_object_no_relro (501,504 samples, 0.32%)_dl_lookup_symbol_x (501,504 samples, 0.32%)clap_builder::parser::parser::Parser::start_custom_arg (499,923 samples, 0.32%)indicatif::progress_bar::ProgressBar::inc (812,875 samples, 0.52%)pcache1Alloc (1,962,395 samples, 1.26%)pcache1EnforceMaxPage (1,247,384 samples, 0.80%)posixUnlock.constprop.0 (706,057 samples, 0.45%)pthreadMutexFree (445,637 samples, 0.29%)realloc (881,828 samples, 0.57%)_int_realloc (881,828 samples, 0.57%)serde_json::ser::format_escaped_str (1,700,364 samples, 1.09%)sqlite3AddPrimaryKey (1,015,414 samples, 0.65%)sqlite3BtreeCommitPhaseOne.part.0 (790,086 samples, 0.51%)sqlite3ExprListAppendNew (1,223,791 samples, 0.79%)sqlite3ExprSetHeightAndFlags.part.0 (1,080,316 samples, 0.69%)sqlite3WhereEnd (1,875,442 samples, 1.20%)std::io::Write::write_all (974,812 samples, 0.63%)unixClose (882,260 samples, 0.57%)all (155,688,399 samples, 100%)db (155,688,399 samples, 100.00%)dbwalIndexAppend (1,322,778 samples, 0.85%) \ No newline at end of file diff --git a/db/src/db.rs b/db/src/db.rs index d59372f..7a7bdf9 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -1,12 +1,55 @@ use diesel::prelude::*; +use diesel::connection::SimpleConnection; +use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool}; use std::env; use crate::models::*; +#[derive(Debug)] +pub struct Fuck (); + +impl CustomizeConnection for Fuck { + fn on_acquire (&self, conn: &mut SqliteConnection) -> Result< + (), diesel::r2d2::Error + > { + conn.batch_execute (" + PRAGMA busy_timeout = 15000; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA wal_autocheckpoint = 1000; + PRAGMA wal_checkpoint(TRUNCATE); + PRAGMA cache_size=-2000000 + ").unwrap (); + Ok (()) + } + + fn on_release (&self, _conn: SqliteConnection) {} +} + +pub fn get_pool () -> Pool> { + let url = env::var ("DATABASE_URL") + .expect ("DATABASE_URL must be set"); + let manager = ConnectionManager::::new (url); + Pool::builder () + .connection_customizer (Box::new (Fuck ())) + .test_on_check_out (true) + .build (manager) + .expect ("Could not build connection pool") +} + pub fn connect () -> SqliteConnection { let database_url = env::var ("DATABASE_URL") .expect ("DATABASE_URL must be set"); - SqliteConnection::establish (&database_url) - .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url)) + let mut conn = SqliteConnection::establish (&database_url) + .unwrap_or_else (|_| panic! ("Error connecting to {}", database_url)); + conn.batch_execute (" + PRAGMA journal_mode = WAL; -- better write-concurrency + PRAGMA synchronous = NORMAL; -- fsync only in critical moments + PRAGMA wal_autocheckpoint = 1000; -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased + PRAGMA wal_checkpoint(TRUNCATE); -- free some space by truncating possibly massive WAL files from the last run. + PRAGMA busy_timeout = 5000; -- sleep if the database is busy + PRAGMA cache_size=-2000000 + ").unwrap (); + conn } pub fn get_word (w : &str) -> Vec { @@ -39,7 +82,6 @@ pub fn insert_word ( }; diesel::insert_into (words::table) .values (new_word) - .execute (conn) - .expect ("error saving new word"); + .execute (conn).ok ()?; Some (()) } diff --git a/db/src/main.rs b/db/src/main.rs index c490705..18fc983 100644 --- a/db/src/main.rs +++ b/db/src/main.rs @@ -4,10 +4,12 @@ pub mod db; use flate2::read::GzDecoder; use indicatif::{ProgressBar, ProgressStyle}; +use indicatif::ParallelProgressIterator; use serde_json::Value; use wiktionary_schema::en::WordData; -use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf}; +use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock}; use clap::Parser; +use rayon::prelude::*; /// Build an SQLite database from a Wiktextract JSONLines dump. #[derive(Parser, Debug)] @@ -40,18 +42,19 @@ fn main() -> Result<(), String> { "{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}" ) .unwrap() ); - let conn = &mut db::connect (); + let lines = d.lines ().map (|l| l.unwrap ().to_owned ()) + .par_bridge () + .progress_with (bar); + let pool = db::get_pool (); - for i in d.lines () { + lines.for_each (|line| { + let conn = &mut pool.clone ().get ().unwrap (); if let Ok (word) = serde_json::from_str:: ( - &i.expect ("entry") + &line ) { - bar.println (db::entry_as_word (&word).map_or ("", |x| x)); db::insert_word (conn, word); - bar.inc (1); } - } + }); - bar.finish (); Ok (()) } diff --git a/db/words.db-journal b/db/words.db-journal deleted file mode 100644 index 5faf9f74fc2106df7705fe3c4e8adf3de1e638ee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12824 zcmZQT0+DE0Uy_pN(tnQoXosbbW>gP(sMG4GZf(RaC1{qiK=u(qWoM1bkp>dbd)ku6I1Z`x++6SM+xNAvecp^bk~7g z1qu_0PeCSuos^T9m#$=`|}ia()UZJirDP`KiP+FXro~n>qk(gVMlZt8>$dzCbh<6lH^GZMtQ~>MC%uCKMDoRZ*Q2_a|6l4e} z7*dNs7C|*290(OCE-6h(%_~tT2I)akh~ys607r#_qWrwlyyQ$+f`W#p0#sR1W=TnE zoN(;Ha5~RHj9nb zQ36Ysq*jz5%bMvJ8XzQ#67!04z9aQeNhvoeF7*d5;>mDWsR?rGt3+hc+vuC*~g7sgRop zlG*yGardE}3Mr{2hjxPWmL!5Qe6&(XQDRCaC?hB4=t9F?Nk_@6vb3ZiGcR2!R!0fs z86_*FrxO=F*}eB^*Zdc&Ha*$h^=#%WP^Ng?(4?fJRD$pl5za2DECA&XXc|-k6&m1B z1A7NKNQP{n2506L~gcJ93!-vPllsYcEaRT=T15(`|Jl^og$PQW04!1M)|B<7_g7NzK-TL&(f zKmiR?iLAiAG%+u|DkHxXE&I{lHe^=>Wad?7=7HRZV(+8I?KJQv*vl#TdFi@Apz<`c z3S20nS$=qB3N(x}3lf#Al=MqZ&OSY%2s^8a99;l07YcY=R0pBC>8 z9EwMUM?+vV1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz7>9fdCdZacynJ$dbgQoYdmN zoXnEc_~O*U($u_UbS|rNkgH>et3rsQlaH$cnpzFe@;II1)I!ZhX=ZkDadE~bQLvfd zjwG01LD3HuQqbU1fPl;th0MH?)b!LMg}nR{g}l<79EF0S%-qDHN`>syN`=JIlKjlP z#ytI4;(7GPDkzAUM5sd8Oj*g5?)?nAg$EReb z7N_RK7bj<=<|fAHW~LW`r{{}t%P^t%1GhSa=fKOX^79nJ9D|%a9D_7W4Kx)3f;@d4 zgCZ5YT_Y9z{6iG{LVbMTE-xz0i_gzf2=VlF4GwYi4S=gqaB+2W4D|_7a1IR$a`g*| xhb!UI3}#{zPj_Tw$jnPgt$;=kB-m0?E8=mxGu{xl95Vw00|zKsMe> r - :out - str/split-lines - (map #(-> % parse-plaintext-entry second)) - (filter #(not (empty? %))) ; remove nil and "" + (let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})] + (with-open [conn (sql/get-connection ds)] + (->> (sql/execute! conn ["select info from words where word = ?" + word]) + (map #(json/parse-string (:words/info %) keyword)) + (mapcat :senses) + (mapcat :raw_glosses) first)))) (defn parse-chat-message [x]