msyds/wiktlarp-tf2
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix-gloss

Browse Source
This commit is contained in:
msyds committed 2026-08-04 19:39:16 -06:00
1 parent c4d4bc8735
commit 6eb7e98864
4 files changed
+43 -503
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

db/flamegraph.svg
-491
View File
@@ -1,491 +0,0 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" width="1200" height="918" onload="init(evt)" viewBox="0 0 1200 918" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:fg="http://github.com/jonhoo/inferno"><!--Flame graph stack visualization. See https://github.com/brendangregg/FlameGraph for latest version, and http://www.brendangregg.com/flamegraphs.html for examples.--><!--NOTES: --><defs><linearGradient id="background" y1="0" y2="1" x1="0" x2="0"><stop stop-color="#eeeeee" offset="5%"/><stop stop-color="#eeeeb0" offset="95%"/></linearGradient></defs><style type="text/css">
text { font-family:monospace; font-size:12px }
#title { text-anchor:middle; font-size:17px; }
#matched { text-anchor:end; }
#search { text-anchor:end; opacity:0.1; cursor:pointer; }
#search:hover, #search.show { opacity:1; }
#subtitle { text-anchor:middle; font-color:rgb(160,160,160); }
#unzoom { cursor:pointer; }
#frames > *:hover { stroke:black; stroke-width:0.5; cursor:pointer; }
.hide { display:none; }
.parent { opacity:0.5; }
</style><script type="text/ecmascript"><![CDATA[
var nametype = 'Function:';
var fontsize = 12;
var fontwidth = 0.59;
var xpad = 10;
var inverted = false;
var searchcolor = 'rgb(230,0,230)';
var fluiddrawing = true;
var truncate_text_right = false;
]]><![CDATA["use strict";
var details, searchbtn, unzoombtn, matchedtxt, svg, searching, frames, known_font_width;
function init(evt) {
details = document.getElementById("details").firstChild;
searchbtn = document.getElementById("search");
unzoombtn = document.getElementById("unzoom");
matchedtxt = document.getElementById("matched");
svg = document.getElementsByTagName("svg")[0];
frames = document.getElementById("frames");
known_font_width = get_monospace_width(frames);
total_samples = parseInt(frames.attributes.total_samples.value);
searching = 0;
// Use GET parameters to restore a flamegraph's state.
var restore_state = function() {
var params = get_params();
if (params.x && params.y)
zoom(find_group(document.querySelector('[*|x="' + params.x + '"][y="' + params.y + '"]')));
if (params.s)
search(params.s);
};
if (fluiddrawing) {
// Make width dynamic so the SVG fits its parent's width.
svg.removeAttribute("width");
// Edge requires us to have a viewBox that gets updated with size changes.
var isEdge = /Edge\/\d./i.test(navigator.userAgent);
if (!isEdge) {
svg.removeAttribute("viewBox");
}
var update_for_width_change = function() {
if (isEdge) {
svg.attributes.viewBox.value = "0 0 " + svg.width.baseVal.value + " " + svg.height.baseVal.value;
}
// Keep consistent padding on left and right of frames container.
frames.attributes.width.value = svg.width.baseVal.value - xpad * 2;
// Text truncation needs to be adjusted for the current width.
update_text_for_elements(frames.children);
// Keep search elements at a fixed distance from right edge.
var svgWidth = svg.width.baseVal.value;
searchbtn.attributes.x.value = svgWidth - xpad;
matchedtxt.attributes.x.value = svgWidth - xpad;
};
window.addEventListener('resize', function() {
update_for_width_change();
});
// This needs to be done asynchronously for Safari to work.
setTimeout(function() {
unzoom();
update_for_width_change();
restore_state();
}, 0);
} else {
restore_state();
}
}
// event listeners
window.addEventListener("click", function(e) {
var target = find_group(e.target);
if (target) {
if (target.nodeName == "a") {
if (e.ctrlKey === false) return;
e.preventDefault();
}
if (target.classList.contains("parent")) unzoom();
zoom(target);
// set parameters for zoom state
var el = target.querySelector("rect");
if (el && el.attributes && el.attributes.y && el.attributes["fg:x"]) {
var params = get_params()
params.x = el.attributes["fg:x"].value;
params.y = el.attributes.y.value;
history.replaceState(null, null, parse_params(params));
}
}
else if (e.target.id == "unzoom") {
unzoom();
// remove zoom state
var params = get_params();
if (params.x) delete params.x;
if (params.y) delete params.y;
history.replaceState(null, null, parse_params(params));
}
else if (e.target.id == "search") search_prompt();
}, false)
// mouse-over for info
// show
window.addEventListener("mouseover", function(e) {
var target = find_group(e.target);
if (target) details.nodeValue = nametype + " " + g_to_text(target);
}, false)
// clear
window.addEventListener("mouseout", function(e) {
var target = find_group(e.target);
if (target) details.nodeValue = ' ';
}, false)
// ctrl-F for search
window.addEventListener("keydown",function (e) {
if (e.keyCode === 114 || (e.ctrlKey && e.keyCode === 70)) {
e.preventDefault();
search_prompt();
}
}, false)
// functions
function get_params() {
var params = {};
var paramsarr = window.location.search.substr(1).split('&');
for (var i = 0; i < paramsarr.length; ++i) {
var tmp = paramsarr[i].split("=");
if (!tmp[0] || !tmp[1]) continue;
params[tmp[0]] = decodeURIComponent(tmp[1]);
}
return params;
}
function parse_params(params) {
var uri = "?";
for (var key in params) {
uri += key + '=' + encodeURIComponent(params[key]) + '&';
}
if (uri.slice(-1) == "&")
uri = uri.substring(0, uri.length - 1);
if (uri == '?')
uri = window.location.href.split('?')[0];
return uri;
}
function find_child(node, selector) {
var children = node.querySelectorAll(selector);
if (children.length) return children[0];
return;
}
function find_group(node) {
var parent = node.parentElement;
if (!parent) return;
if (parent.id == "frames") return node;
return find_group(parent);
}
function orig_save(e, attr, val) {
if (e.attributes["fg:orig_" + attr] != undefined) return;
if (e.attributes[attr] == undefined) return;
if (val == undefined) val = e.attributes[attr].value;
e.setAttribute("fg:orig_" + attr, val);
}
function orig_load(e, attr) {
if (e.attributes["fg:orig_"+attr] == undefined) return;
e.attributes[attr].value = e.attributes["fg:orig_" + attr].value;
e.removeAttribute("fg:orig_" + attr);
}
function g_to_text(e) {
var text = find_child(e, "title").firstChild.nodeValue;
return (text)
}
function g_to_func(e) {
var func = g_to_text(e);
// if there's any manipulation we want to do to the function
// name before it's searched, do it here before returning.
return (func);
}
function get_monospace_width(frames) {
// Given the id="frames" element, return the width of text characters if
// this is a monospace font, otherwise return 0.
text = find_child(frames.children[0], "text");
originalContent = text.textContent;
text.textContent = "!";
bangWidth = text.getComputedTextLength();
text.textContent = "W";
wWidth = text.getComputedTextLength();
text.textContent = originalContent;
if (bangWidth === wWidth) {
return bangWidth;
} else {
return 0;
}
}
function update_text_for_elements(elements) {
// In order to render quickly in the browser, you want to do one pass of
// reading attributes, and one pass of mutating attributes. See
// https://web.dev/avoid-large-complex-layouts-and-layout-thrashing/ for details.
// Fall back to inefficient calculation, if we're variable-width font.
// TODO This should be optimized somehow too.
if (known_font_width === 0) {
for (var i = 0; i < elements.length; i++) {
update_text(elements[i]);
}
return;
}
var textElemNewAttributes = [];
for (var i = 0; i < elements.length; i++) {
var e = elements[i];
var r = find_child(e, "rect");
var t = find_child(e, "text");
var w = parseFloat(r.attributes.width.value) * frames.attributes.width.value / 100 - 3;
var txt = find_child(e, "title").textContent.replace(/\([^(]*\)$/,"");
var newX = format_percent((parseFloat(r.attributes.x.value) + (100 * 3 / frames.attributes.width.value)));
// Smaller than this size won't fit anything
if (w < 2 * known_font_width) {
textElemNewAttributes.push([newX, ""]);
continue;
}
// Fit in full text width
if (txt.length * known_font_width < w) {
textElemNewAttributes.push([newX, txt]);
continue;
}
var substringLength = Math.floor(w / known_font_width) - 2;
if (truncate_text_right) {
// Truncate the right side of the text.
textElemNewAttributes.push([newX, txt.substring(0, substringLength) + ".."]);
continue;
} else {
// Truncate the left side of the text.
textElemNewAttributes.push([newX, ".." + txt.substring(txt.length - substringLength, txt.length)]);
continue;
}
}
console.assert(textElemNewAttributes.length === elements.length, "Resize failed, please file a bug at https://github.com/jonhoo/inferno/");
// Now that we know new textContent, set it all in one go so we don't refresh a bazillion times.
for (var i = 0; i < elements.length; i++) {
var e = elements[i];
var values = textElemNewAttributes[i];
var t = find_child(e, "text");
t.attributes.x.value = values[0];
t.textContent = values[1];
}
}
function update_text(e) {
var r = find_child(e, "rect");
var t = find_child(e, "text");
var w = parseFloat(r.attributes.width.value) * frames.attributes.width.value / 100 - 3;
var txt = find_child(e, "title").textContent.replace(/\([^(]*\)$/,"");
t.attributes.x.value = format_percent((parseFloat(r.attributes.x.value) + (100 * 3 / frames.attributes.width.value)));
// Smaller than this size won't fit anything
if (w < 2 * fontsize * fontwidth) {
t.textContent = "";
return;
}
t.textContent = txt;
// Fit in full text width
if (t.getComputedTextLength() < w)
return;
if (truncate_text_right) {
// Truncate the right side of the text.
for (var x = txt.length - 2; x > 0; x--) {
if (t.getSubStringLength(0, x + 2) <= w) {
t.textContent = txt.substring(0, x) + "..";
return;
}
}
} else {
// Truncate the left side of the text.
for (var x = 2; x < txt.length; x++) {
if (t.getSubStringLength(x - 2, txt.length) <= w) {
t.textContent = ".." + txt.substring(x, txt.length);
return;
}
}
}
t.textContent = "";
}
// zoom
function zoom_reset(e) {
if (e.tagName == "rect") {
e.attributes.x.value = format_percent(100 * parseInt(e.attributes["fg:x"].value) / total_samples);
e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / total_samples);
}
if (e.childNodes == undefined) return;
for(var i = 0, c = e.childNodes; i < c.length; i++) {
zoom_reset(c[i]);
}
}
function zoom_child(e, x, zoomed_width_samples) {
if (e.tagName == "text") {
var parent_x = parseFloat(find_child(e.parentNode, "rect[x]").attributes.x.value);
e.attributes.x.value = format_percent(parent_x + (100 * 3 / frames.attributes.width.value));
} else if (e.tagName == "rect") {
e.attributes.x.value = format_percent(100 * (parseInt(e.attributes["fg:x"].value) - x) / zoomed_width_samples);
e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / zoomed_width_samples);
}
if (e.childNodes == undefined) return;
for(var i = 0, c = e.childNodes; i < c.length; i++) {
zoom_child(c[i], x, zoomed_width_samples);
}
}
function zoom_parent(e) {
if (e.attributes) {
if (e.attributes.x != undefined) {
e.attributes.x.value = "0.0%";
}
if (e.attributes.width != undefined) {
e.attributes.width.value = "100.0%";
}
}
if (e.childNodes == undefined) return;
for(var i = 0, c = e.childNodes; i < c.length; i++) {
zoom_parent(c[i]);
}
}
function zoom(node) {
var attr = find_child(node, "rect").attributes;
var width = parseInt(attr["fg:w"].value);
var xmin = parseInt(attr["fg:x"].value);
var xmax = xmin + width;
var ymin = parseFloat(attr.y.value);
unzoombtn.classList.remove("hide");
var el = frames.children;
var to_update_text = [];
for (var i = 0; i < el.length; i++) {
var e = el[i];
var a = find_child(e, "rect").attributes;
var ex = parseInt(a["fg:x"].value);
var ew = parseInt(a["fg:w"].value);
// Is it an ancestor
if (!inverted) {
var upstack = parseFloat(a.y.value) > ymin;
} else {
var upstack = parseFloat(a.y.value) < ymin;
}
if (upstack) {
// Direct ancestor
if (ex <= xmin && (ex+ew) >= xmax) {
e.classList.add("parent");
zoom_parent(e);
to_update_text.push(e);
}
// not in current path
else
e.classList.add("hide");
}
// Children maybe
else {
// no common path
if (ex < xmin || ex >= xmax) {
e.classList.add("hide");
}
else {
zoom_child(e, xmin, width);
to_update_text.push(e);
}
}
}
update_text_for_elements(to_update_text);
}
function unzoom() {
unzoombtn.classList.add("hide");
var el = frames.children;
for(var i = 0; i < el.length; i++) {
el[i].classList.remove("parent");
el[i].classList.remove("hide");
zoom_reset(el[i]);
}
update_text_for_elements(el);
}
// search
function reset_search() {
var el = document.querySelectorAll("#frames rect");
for (var i = 0; i < el.length; i++) {
orig_load(el[i], "fill")
}
var params = get_params();
delete params.s;
history.replaceState(null, null, parse_params(params));
}
function search_prompt() {
if (!searching) {
var term = prompt("Enter a search term (regexp " +
"allowed, eg: ^ext4_)", "");
if (term != null) {
search(term)
}
} else {
reset_search();
searching = 0;
searchbtn.classList.remove("show");
searchbtn.firstChild.nodeValue = "Search"
matchedtxt.classList.add("hide");
matchedtxt.firstChild.nodeValue = ""
}
}
function search(term) {
var re = new RegExp(term);
var el = frames.children;
var matches = new Object();
var maxwidth = 0;
for (var i = 0; i < el.length; i++) {
var e = el[i];
// Skip over frames which are either not visible, or below the zoomed-to frame
if (e.classList.contains("hide") || e.classList.contains("parent")) {
continue;
}
var func = g_to_func(e);
var rect = find_child(e, "rect");
if (func == null || rect == null)
continue;
// Save max width. Only works as we have a root frame
var w = parseInt(rect.attributes["fg:w"].value);
if (w > maxwidth)
maxwidth = w;
if (func.match(re)) {
// highlight
var x = parseInt(rect.attributes["fg:x"].value);
orig_save(rect, "fill");
rect.attributes.fill.value = searchcolor;
// remember matches
if (matches[x] == undefined) {
matches[x] = w;
} else {
if (w > matches[x]) {
// overwrite with parent
matches[x] = w;
}
}
searching = 1;
}
}
if (!searching)
return;
var params = get_params();
params.s = term;
history.replaceState(null, null, parse_params(params));
searchbtn.classList.add("show");
searchbtn.firstChild.nodeValue = "Reset Search";
// calculate percent matched, excluding vertical overlap
var count = 0;
var lastx = -1;
var lastw = 0;
var keys = Array();
for (k in matches) {
if (matches.hasOwnProperty(k))
keys.push(k);
}
// sort the matched frames by their x location
// ascending, then width descending
keys.sort(function(a, b){
return a - b;
});
// Step through frames saving only the biggest bottom-up frames
// thanks to the sort order. This relies on the tree property
// where children are always smaller than their parents.
for (var k in keys) {
var x = parseInt(keys[k]);
var w = matches[keys[k]];
if (x >= lastx + lastw) {
count += w;
lastx = x;
lastw = w;
}
}
// display matched percent
matchedtxt.classList.remove("hide");
var pct = 100 * count / maxwidth;
if (pct != 100) pct = pct.toFixed(1);
matchedtxt.firstChild.nodeValue = "Matched: " + pct + "%";
}
function format_percent(n) {
return n.toFixed(4) + "%";
}
]]></script><rect x="0" y="0" width="100%" height="918" fill="url(#background)"/><text id="title" fill="rgb(0,0,0)" x="50.0000%" y="24.00">Flame Graph</text><text id="details" fill="rgb(0,0,0)" x="10" y="901.00"> </text><text id="unzoom" class="hide" fill="rgb(0,0,0)" x="10" y="24.00">Reset Zoom</text><text id="search" fill="rgb(0,0,0)" x="1190" y="24.00">Search</text><text id="matched" fill="rgb(0,0,0)" x="1190" y="901.00"> </text><svg id="frames" x="10" width="1180" total_samples="155688399"><g><title>&lt;alloc::string::String as core::fmt::Write&gt;::write_str (1,932,904 samples, 1.24%)</title><rect x="0.0000%" y="837" width="1.2415%" height="15" fill="rgb(227,0,7)" fg:x="0" fg:w="1932904"/><text x="0.2500%" y="847.50"></text></g><g><title>&lt;console::utils::StyledObject&lt;D&gt; as core::fmt::Display&gt;::fmt (501,504 samples, 0.32%)</title><rect x="1.2415%" y="837" width="0.3221%" height="15" fill="rgb(217,0,24)" fg:x="1932904" fg:w="501504"/><text x="1.4915%" y="847.50"></text></g><g><title>&lt;diesel::sqlite::connection::SqliteConnection as diesel::connection::Connection&gt;::establish (1,202,489 samples, 0.77%)</title><rect x="1.5636%" y="837" width="0.7724%" height="15" fill="rgb(221,193,54)" fg:x="2434408" fg:w="1202489"/><text x="1.8136%" y="847.50"></text></g><g><title>&lt;diesel::sqlite::query_builder::SqliteQueryBuilder as diesel::query_builder::QueryBuilder&lt;diesel::sqlite::backend::Sqlite&gt;&gt;::push_sql (1,045,240 samples, 0.67%)</title><rect x="2.3360%" y="837" width="0.6714%" height="15" fill="rgb(248,212,6)" fg:x="3636897" fg:w="1045240"/><text x="2.5860%" y="847.50"></text></g><g><title>[db] (652,185 samples, 0.42%)</title><rect x="3.0074%" y="837" width="0.4189%" height="15" fill="rgb(208,68,35)" fg:x="4682137" fg:w="652185"/><text x="3.2574%" y="847.50"></text></g><g><title>std::io::Write::write_all (652,185 samples, 0.42%)</title><rect x="3.0074%" y="821" width="0.4189%" height="15" fill="rgb(232,128,0)" fg:x="4682137" fg:w="652185"/><text x="3.2574%" y="831.50"></text></g><g><title>&lt;serde_json::read::StrRead as serde_json::read::Read&gt;::parse_str (880,273 samples, 0.57%)</title><rect x="3.4263%" y="821" width="0.5654%" height="15" fill="rgb(207,160,47)" fg:x="5334322" fg:w="880273"/><text x="3.6763%" y="831.50"></text></g><g><title>[unknown] (1,181,670 samples, 0.76%)</title><rect x="3.9917%" y="821" width="0.7590%" height="15" fill="rgb(228,23,34)" fg:x="6214595" fg:w="1181670"/><text x="4.2417%" y="831.50"></text></g><g><title>serde_json::value::de::&lt;impl serde_core::de::Deserialize for serde_json::value::Value&gt;::deserialize (1,181,670 samples, 0.76%)</title><rect x="3.9917%" y="805" width="0.7590%" height="15" fill="rgb(218,30,26)" fg:x="6214595" fg:w="1181670"/><text x="4.2417%" y="815.50"></text></g><g><title>[unknown] (2,563,447 samples, 1.65%)</title><rect x="3.4263%" y="837" width="1.6465%" height="15" fill="rgb(220,122,19)" fg:x="5334322" fg:w="2563447"/><text x="3.6763%" y="847.50"></text></g><g><title>indicatif::state::ProgressState::eta (501,504 samples, 0.32%)</title><rect x="4.7507%" y="821" width="0.3221%" height="15" fill="rgb(250,228,42)" fg:x="7396265" fg:w="501504"/><text x="5.0007%" y="831.50"></text></g><g><title>&lt;diesel::sqlite::connection::SqliteConnection as diesel::connection::SimpleConnection&gt;::batch_execute (829,243 samples, 0.53%)</title><rect x="5.0728%" y="645" width="0.5326%" height="15" fill="rgb(240,193,28)" fg:x="7897769" fg:w="829243"/><text x="5.3228%" y="655.50"></text></g><g><title>diesel::sqlite::connection::raw::RawConnection::exec (829,243 samples, 0.53%)</title><rect x="5.0728%" y="629" width="0.5326%" height="15" fill="rgb(216,20,37)" fg:x="7897769" fg:w="829243"/><text x="5.3228%" y="639.50"></text></g><g><title>sqlite3_prepare_v2 (829,243 samples, 0.53%)</title><rect x="5.0728%" y="613" width="0.5326%" height="15" fill="rgb(206,188,39)" fg:x="7897769" fg:w="829243"/><text x="5.3228%" y="623.50"></text></g><g><title>&lt;indicatif::rayon::ProgressFolder&lt;C&gt; as rayon::iter::plumbing::Folder&lt;T&gt;&gt;::consume (1,330,747 samples, 0.85%)</title><rect x="5.0728%" y="677" width="0.8548%" height="15" fill="rgb(217,207,13)" fg:x="7897769" fg:w="1330747"/><text x="5.3228%" y="687.50"></text></g><g><title>db::db::connect (1,330,747 samples, 0.85%)</title><rect x="5.0728%" y="661" width="0.8548%" height="15" fill="rgb(231,73,38)" fg:x="7897769" fg:w="1330747"/><text x="5.3228%" y="671.50"></text></g><g><title>core::result::unwrap_failed (501,504 samples, 0.32%)</title><rect x="5.6054%" y="645" width="0.3221%" height="15" fill="rgb(225,20,46)" fg:x="8727012" fg:w="501504"/><text x="5.8554%" y="655.50"></text></g><g><title>core::panicking::panic_fmt (501,504 samples, 0.32%)</title><rect x="5.6054%" y="629" width="0.3221%" height="15" fill="rgb(210,31,41)" fg:x="8727012" fg:w="501504"/><text x="5.8554%" y="639.50"></text></g><g><title>__rustc::rust_begin_unwind (501,504 samples, 0.32%)</title><rect x="5.6054%" y="613" widtLine truncated
Side by Side

Before

Width:  |  Height:  |  Size: 113 KiB

db/src/db.rs
+9 -2
View File
@@ -76,12 +76,19 @@ pub fn insert_word (
entry : serde_json::Value,
) -> Option<()> {
use crate::schema::words;
let word = entry_as_word (&entry)?;
let new_word = NewWord {
word: entry_as_word (&entry)?,
word,
info: &entry.to_string (),
};
diesel::insert_into (words::table)
.values (new_word)
.execute (conn).ok ()?;
.execute (conn).map_or_else (
|e| {
println! ("error for {}: {}", word, e);
None
},
Some
)?;
Some (())
}
db/src/main.rs
+24 -4
View File
@@ -2,6 +2,8 @@ pub mod models;
pub mod schema;
pub mod db;
use diesel::SqliteConnection;
use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
use flate2::read::GzDecoder;
use indicatif::{ProgressBar, ProgressStyle};
use indicatif::ParallelProgressIterator;
@@ -33,6 +35,18 @@ fn open_wiktextract_dump (path : PathBuf) -> Result<
#[allow(nonstandard_style)]
const wiktextract_dump_lines : u64 = 10736399;
fn do_line (
conn: &mut PooledConnection<ConnectionManager<SqliteConnection>>,
retries : &RwLock<Vec<String>>,
line : String
) {
if let Ok (word) = serde_json::from_str::<Value> (&line) {
if let None = db::insert_word (conn, word) {
retries.write ().unwrap ().push (line);
}
}
}
fn main() -> Result<(), String> {
let args = Args::parse ();
let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
@@ -46,15 +60,21 @@ fn main() -> Result<(), String> {
.par_bridge ()
.progress_with (bar);
let pool = db::get_pool ();
let retries = RwLock::new (vec! []);
lines.for_each (|line| {
let conn = &mut pool.clone ().get ().unwrap ();
if let Ok (word) = serde_json::from_str::<Value> (
&line
) {
do_line (conn, &retries, line)
});
println! ("do retries now");
let conn = &mut pool.clone ().get ().unwrap ();
for line in retries.read ().unwrap ().iter () {
println! ("retrying");
if let Ok (word) = serde_json::from_str::<Value> (&line) {
db::insert_word (conn, word);
}
});
}
Ok (())
}
src/wiktionary_tf2/main.clj
+10 -6
View File
@@ -22,15 +22,19 @@
(when-some [[_ word gloss] (re-matches #"([^\t]*)\t(.*)\n?" s)]
[word gloss]))
(defn lookup-word [word]
(defn query-word [word]
(let [ds (sql/get-datasource {:dbtype "sqlite" :dbname "db/words.db"})]
(with-open [conn (sql/get-connection ds)]
(->> (sql/execute! conn ["select info from words where word = ?"
word])
(map #(json/parse-string (:words/info %) keyword))
(mapcat :senses)
(mapcat :raw_glosses)
first))))
(map #(json/parse-string (:words/info %) keyword))))))
(defn gloss-word [word]
(->> word
query-word
(mapcat :senses)
(mapcat (some-fn :raw_glosses :glosses))
first))
(defn parse-chat-message [x]
(when-let [[_ _dead? author body]
@@ -52,7 +56,7 @@
(defn do-definition [s]
(when-some [w (find-word s)]
(l/infof "looking up word: %s" w)
(let [r (lookup-word w)]
(let [r (gloss-word w)]
(if r
(l/infof "found word! %s" r)
(l/infof "no entry... %s" w))