excellent
This commit is contained in:
msyds committed
1 parent
05d48c0c3b
commit
c4d4bc8735
9 files changed
+695
-25
No files matched your search
+8
-1
@@ -1 +1,8 @@
|
||||
target
|
||||
raw-wiktextract-data.jsonl.gz
|
||||
target/
|
||||
words.db
|
||||
words.db-journal
|
||||
words.db-wal
|
||||
words.db-shm
|
||||
words.preview.db
|
||||
flamegraph.svg
|
||||
Generated
+127
@@ -168,6 +168,31 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.21.3"
|
||||
@@ -212,6 +237,7 @@ dependencies = [
|
||||
"diesel",
|
||||
"flate2",
|
||||
"indicatif",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"wiktionary-schema",
|
||||
@@ -232,6 +258,7 @@ dependencies = [
|
||||
"diesel_derives",
|
||||
"downcast-rs",
|
||||
"libsqlite3-sys",
|
||||
"r2d2",
|
||||
"sqlite-wasm-rs",
|
||||
"time",
|
||||
]
|
||||
@@ -405,6 +432,7 @@ checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c"
|
||||
dependencies = [
|
||||
"console",
|
||||
"portable-atomic",
|
||||
"rayon",
|
||||
"unicode-width",
|
||||
"unit-prefix",
|
||||
"web-time",
|
||||
@@ -466,6 +494,21 @@ version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
@@ -500,6 +543,29 @@ version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
@@ -548,6 +614,46 @@ version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "r2d2"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
|
||||
dependencies = [
|
||||
"log",
|
||||
"parking_lot",
|
||||
"scheduled-thread-pool",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rsqlite-vfs"
|
||||
version = "0.1.1"
|
||||
@@ -586,6 +692,21 @@ dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scheduled-thread-pool"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
|
||||
dependencies = [
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
@@ -647,6 +768,12 @@ version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-wasm-rs"
|
||||
version = "0.5.5"
|
||||
|
||||
+3
-2
@@ -6,9 +6,10 @@ edition = "2024"
|
||||
[dependencies]
|
||||
clap = { version = "4.6.5", features = ["derive"] }
|
||||
clio = { version = "0.3.5", features = ["clap-parse"] }
|
||||
diesel = { version = "2.3.11", features = ["sqlite"] }
|
||||
diesel = { version = "2.3.11", features = ["r2d2", "sqlite"] }
|
||||
flate2 = { version = "1.1.9", features = ["zlib-rs"] }
|
||||
indicatif = "0.18.6"
|
||||
indicatif = { version = "0.18.6", features = ["rayon"] }
|
||||
rayon = "1.12.0"
|
||||
serde = "1.0.229"
|
||||
serde_json = "1.0.151"
|
||||
wiktionary-schema = "0.3.0"
|
||||
@@ -0,0 +1,491 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" width="1200" height="918" onload="init(evt)" viewBox="0 0 1200 918" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:fg="http://github.com/jonhoo/inferno"><!--Flame graph stack visualization. See https://github.com/brendangregg/FlameGraph for latest version, and http://www.brendangregg.com/flamegraphs.html for examples.--><!--NOTES: --><defs><linearGradient id="background" y1="0" y2="1" x1="0" x2="0"><stop stop-color="#eeeeee" offset="5%"/><stop stop-color="#eeeeb0" offset="95%"/></linearGradient></defs><style type="text/css">
|
||||
text { font-family:monospace; font-size:12px }
|
||||
#title { text-anchor:middle; font-size:17px; }
|
||||
#matched { text-anchor:end; }
|
||||
#search { text-anchor:end; opacity:0.1; cursor:pointer; }
|
||||
#search:hover, #search.show { opacity:1; }
|
||||
#subtitle { text-anchor:middle; font-color:rgb(160,160,160); }
|
||||
#unzoom { cursor:pointer; }
|
||||
#frames > *:hover { stroke:black; stroke-width:0.5; cursor:pointer; }
|
||||
.hide { display:none; }
|
||||
.parent { opacity:0.5; }
|
||||
</style><script type="text/ecmascript"><![CDATA[
|
||||
var nametype = 'Function:';
|
||||
var fontsize = 12;
|
||||
var fontwidth = 0.59;
|
||||
var xpad = 10;
|
||||
var inverted = false;
|
||||
var searchcolor = 'rgb(230,0,230)';
|
||||
var fluiddrawing = true;
|
||||
var truncate_text_right = false;
|
||||
]]><![CDATA["use strict";
|
||||
var details, searchbtn, unzoombtn, matchedtxt, svg, searching, frames, known_font_width;
|
||||
function init(evt) {
|
||||
details = document.getElementById("details").firstChild;
|
||||
searchbtn = document.getElementById("search");
|
||||
unzoombtn = document.getElementById("unzoom");
|
||||
matchedtxt = document.getElementById("matched");
|
||||
svg = document.getElementsByTagName("svg")[0];
|
||||
frames = document.getElementById("frames");
|
||||
known_font_width = get_monospace_width(frames);
|
||||
total_samples = parseInt(frames.attributes.total_samples.value);
|
||||
searching = 0;
|
||||
|
||||
// Use GET parameters to restore a flamegraph's state.
|
||||
var restore_state = function() {
|
||||
var params = get_params();
|
||||
if (params.x && params.y)
|
||||
zoom(find_group(document.querySelector('[*|x="' + params.x + '"][y="' + params.y + '"]')));
|
||||
if (params.s)
|
||||
search(params.s);
|
||||
};
|
||||
|
||||
if (fluiddrawing) {
|
||||
// Make width dynamic so the SVG fits its parent's width.
|
||||
svg.removeAttribute("width");
|
||||
// Edge requires us to have a viewBox that gets updated with size changes.
|
||||
var isEdge = /Edge\/\d./i.test(navigator.userAgent);
|
||||
if (!isEdge) {
|
||||
svg.removeAttribute("viewBox");
|
||||
}
|
||||
var update_for_width_change = function() {
|
||||
if (isEdge) {
|
||||
svg.attributes.viewBox.value = "0 0 " + svg.width.baseVal.value + " " + svg.height.baseVal.value;
|
||||
}
|
||||
|
||||
// Keep consistent padding on left and right of frames container.
|
||||
frames.attributes.width.value = svg.width.baseVal.value - xpad * 2;
|
||||
|
||||
// Text truncation needs to be adjusted for the current width.
|
||||
update_text_for_elements(frames.children);
|
||||
|
||||
// Keep search elements at a fixed distance from right edge.
|
||||
var svgWidth = svg.width.baseVal.value;
|
||||
searchbtn.attributes.x.value = svgWidth - xpad;
|
||||
matchedtxt.attributes.x.value = svgWidth - xpad;
|
||||
};
|
||||
window.addEventListener('resize', function() {
|
||||
update_for_width_change();
|
||||
});
|
||||
// This needs to be done asynchronously for Safari to work.
|
||||
setTimeout(function() {
|
||||
unzoom();
|
||||
update_for_width_change();
|
||||
restore_state();
|
||||
}, 0);
|
||||
} else {
|
||||
restore_state();
|
||||
}
|
||||
}
|
||||
// event listeners
|
||||
window.addEventListener("click", function(e) {
|
||||
var target = find_group(e.target);
|
||||
if (target) {
|
||||
if (target.nodeName == "a") {
|
||||
if (e.ctrlKey === false) return;
|
||||
e.preventDefault();
|
||||
}
|
||||
if (target.classList.contains("parent")) unzoom();
|
||||
zoom(target);
|
||||
|
||||
// set parameters for zoom state
|
||||
var el = target.querySelector("rect");
|
||||
if (el && el.attributes && el.attributes.y && el.attributes["fg:x"]) {
|
||||
var params = get_params()
|
||||
params.x = el.attributes["fg:x"].value;
|
||||
params.y = el.attributes.y.value;
|
||||
history.replaceState(null, null, parse_params(params));
|
||||
}
|
||||
}
|
||||
else if (e.target.id == "unzoom") {
|
||||
unzoom();
|
||||
|
||||
// remove zoom state
|
||||
var params = get_params();
|
||||
if (params.x) delete params.x;
|
||||
if (params.y) delete params.y;
|
||||
history.replaceState(null, null, parse_params(params));
|
||||
}
|
||||
else if (e.target.id == "search") search_prompt();
|
||||
}, false)
|
||||
// mouse-over for info
|
||||
// show
|
||||
window.addEventListener("mouseover", function(e) {
|
||||
var target = find_group(e.target);
|
||||
if (target) details.nodeValue = nametype + " " + g_to_text(target);
|
||||
}, false)
|
||||
// clear
|
||||
window.addEventListener("mouseout", function(e) {
|
||||
var target = find_group(e.target);
|
||||
if (target) details.nodeValue = ' ';
|
||||
}, false)
|
||||
// ctrl-F for search
|
||||
window.addEventListener("keydown",function (e) {
|
||||
if (e.keyCode === 114 || (e.ctrlKey && e.keyCode === 70)) {
|
||||
e.preventDefault();
|
||||
search_prompt();
|
||||
}
|
||||
}, false)
|
||||
// functions
|
||||
function get_params() {
|
||||
var params = {};
|
||||
var paramsarr = window.location.search.substr(1).split('&');
|
||||
for (var i = 0; i < paramsarr.length; ++i) {
|
||||
var tmp = paramsarr[i].split("=");
|
||||
if (!tmp[0] || !tmp[1]) continue;
|
||||
params[tmp[0]] = decodeURIComponent(tmp[1]);
|
||||
}
|
||||
return params;
|
||||
}
|
||||
function parse_params(params) {
|
||||
var uri = "?";
|
||||
for (var key in params) {
|
||||
uri += key + '=' + encodeURIComponent(params[key]) + '&';
|
||||
}
|
||||
if (uri.slice(-1) == "&")
|
||||
uri = uri.substring(0, uri.length - 1);
|
||||
if (uri == '?')
|
||||
uri = window.location.href.split('?')[0];
|
||||
return uri;
|
||||
}
|
||||
function find_child(node, selector) {
|
||||
var children = node.querySelectorAll(selector);
|
||||
if (children.length) return children[0];
|
||||
return;
|
||||
}
|
||||
function find_group(node) {
|
||||
var parent = node.parentElement;
|
||||
if (!parent) return;
|
||||
if (parent.id == "frames") return node;
|
||||
return find_group(parent);
|
||||
}
|
||||
function orig_save(e, attr, val) {
|
||||
if (e.attributes["fg:orig_" + attr] != undefined) return;
|
||||
if (e.attributes[attr] == undefined) return;
|
||||
if (val == undefined) val = e.attributes[attr].value;
|
||||
e.setAttribute("fg:orig_" + attr, val);
|
||||
}
|
||||
function orig_load(e, attr) {
|
||||
if (e.attributes["fg:orig_"+attr] == undefined) return;
|
||||
e.attributes[attr].value = e.attributes["fg:orig_" + attr].value;
|
||||
e.removeAttribute("fg:orig_" + attr);
|
||||
}
|
||||
function g_to_text(e) {
|
||||
var text = find_child(e, "title").firstChild.nodeValue;
|
||||
return (text)
|
||||
}
|
||||
function g_to_func(e) {
|
||||
var func = g_to_text(e);
|
||||
// if there's any manipulation we want to do to the function
|
||||
// name before it's searched, do it here before returning.
|
||||
return (func);
|
||||
}
|
||||
function get_monospace_width(frames) {
|
||||
// Given the id="frames" element, return the width of text characters if
|
||||
// this is a monospace font, otherwise return 0.
|
||||
text = find_child(frames.children[0], "text");
|
||||
originalContent = text.textContent;
|
||||
text.textContent = "!";
|
||||
bangWidth = text.getComputedTextLength();
|
||||
text.textContent = "W";
|
||||
wWidth = text.getComputedTextLength();
|
||||
text.textContent = originalContent;
|
||||
if (bangWidth === wWidth) {
|
||||
return bangWidth;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
function update_text_for_elements(elements) {
|
||||
// In order to render quickly in the browser, you want to do one pass of
|
||||
// reading attributes, and one pass of mutating attributes. See
|
||||
// https://web.dev/avoid-large-complex-layouts-and-layout-thrashing/ for details.
|
||||
|
||||
// Fall back to inefficient calculation, if we're variable-width font.
|
||||
// TODO This should be optimized somehow too.
|
||||
if (known_font_width === 0) {
|
||||
for (var i = 0; i < elements.length; i++) {
|
||||
update_text(elements[i]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
var textElemNewAttributes = [];
|
||||
for (var i = 0; i < elements.length; i++) {
|
||||
var e = elements[i];
|
||||
var r = find_child(e, "rect");
|
||||
var t = find_child(e, "text");
|
||||
var w = parseFloat(r.attributes.width.value) * frames.attributes.width.value / 100 - 3;
|
||||
var txt = find_child(e, "title").textContent.replace(/\([^(]*\)$/,"");
|
||||
var newX = format_percent((parseFloat(r.attributes.x.value) + (100 * 3 / frames.attributes.width.value)));
|
||||
|
||||
// Smaller than this size won't fit anything
|
||||
if (w < 2 * known_font_width) {
|
||||
textElemNewAttributes.push([newX, ""]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fit in full text width
|
||||
if (txt.length * known_font_width < w) {
|
||||
textElemNewAttributes.push([newX, txt]);
|
||||
continue;
|
||||
}
|
||||
|
||||
var substringLength = Math.floor(w / known_font_width) - 2;
|
||||
if (truncate_text_right) {
|
||||
// Truncate the right side of the text.
|
||||
textElemNewAttributes.push([newX, txt.substring(0, substringLength) + ".."]);
|
||||
continue;
|
||||
} else {
|
||||
// Truncate the left side of the text.
|
||||
textElemNewAttributes.push([newX, ".." + txt.substring(txt.length - substringLength, txt.length)]);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
console.assert(textElemNewAttributes.length === elements.length, "Resize failed, please file a bug at https://github.com/jonhoo/inferno/");
|
||||
|
||||
// Now that we know new textContent, set it all in one go so we don't refresh a bazillion times.
|
||||
for (var i = 0; i < elements.length; i++) {
|
||||
var e = elements[i];
|
||||
var values = textElemNewAttributes[i];
|
||||
var t = find_child(e, "text");
|
||||
t.attributes.x.value = values[0];
|
||||
t.textContent = values[1];
|
||||
}
|
||||
}
|
||||
|
||||
function update_text(e) {
|
||||
var r = find_child(e, "rect");
|
||||
var t = find_child(e, "text");
|
||||
var w = parseFloat(r.attributes.width.value) * frames.attributes.width.value / 100 - 3;
|
||||
var txt = find_child(e, "title").textContent.replace(/\([^(]*\)$/,"");
|
||||
t.attributes.x.value = format_percent((parseFloat(r.attributes.x.value) + (100 * 3 / frames.attributes.width.value)));
|
||||
|
||||
// Smaller than this size won't fit anything
|
||||
if (w < 2 * fontsize * fontwidth) {
|
||||
t.textContent = "";
|
||||
return;
|
||||
}
|
||||
t.textContent = txt;
|
||||
// Fit in full text width
|
||||
if (t.getComputedTextLength() < w)
|
||||
return;
|
||||
if (truncate_text_right) {
|
||||
// Truncate the right side of the text.
|
||||
for (var x = txt.length - 2; x > 0; x--) {
|
||||
if (t.getSubStringLength(0, x + 2) <= w) {
|
||||
t.textContent = txt.substring(0, x) + "..";
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Truncate the left side of the text.
|
||||
for (var x = 2; x < txt.length; x++) {
|
||||
if (t.getSubStringLength(x - 2, txt.length) <= w) {
|
||||
t.textContent = ".." + txt.substring(x, txt.length);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
t.textContent = "";
|
||||
}
|
||||
// zoom
|
||||
function zoom_reset(e) {
|
||||
if (e.tagName == "rect") {
|
||||
e.attributes.x.value = format_percent(100 * parseInt(e.attributes["fg:x"].value) / total_samples);
|
||||
e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / total_samples);
|
||||
}
|
||||
if (e.childNodes == undefined) return;
|
||||
for(var i = 0, c = e.childNodes; i < c.length; i++) {
|
||||
zoom_reset(c[i]);
|
||||
}
|
||||
}
|
||||
function zoom_child(e, x, zoomed_width_samples) {
|
||||
if (e.tagName == "text") {
|
||||
var parent_x = parseFloat(find_child(e.parentNode, "rect[x]").attributes.x.value);
|
||||
e.attributes.x.value = format_percent(parent_x + (100 * 3 / frames.attributes.width.value));
|
||||
} else if (e.tagName == "rect") {
|
||||
e.attributes.x.value = format_percent(100 * (parseInt(e.attributes["fg:x"].value) - x) / zoomed_width_samples);
|
||||
e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / zoomed_width_samples);
|
||||
}
|
||||
if (e.childNodes == undefined) return;
|
||||
for(var i = 0, c = e.childNodes; i < c.length; i++) {
|
||||
zoom_child(c[i], x, zoomed_width_samples);
|
||||
}
|
||||
}
|
||||
function zoom_parent(e) {
|
||||
if (e.attributes) {
|
||||
if (e.attributes.x != undefined) {
|
||||
e.attributes.x.value = "0.0%";
|
||||
}
|
||||
if (e.attributes.width != undefined) {
|
||||
e.attributes.width.value = "100.0%";
|
||||
}
|
||||
}
|
||||
if (e.childNodes == undefined) return;
|
||||
for(var i = 0, c = e.childNodes; i < c.length; i++) {
|
||||
zoom_parent(c[i]);
|
||||
}
|
||||
}
|
||||
function zoom(node) {
|
||||
var attr = find_child(node, "rect").attributes;
|
||||
var width = parseInt(attr["fg:w"].value);
|
||||
var xmin = parseInt(attr["fg:x"].value);
|
||||
var xmax = xmin + width;
|
||||
var ymin = parseFloat(attr.y.value);
|
||||
unzoombtn.classList.remove("hide");
|
||||
var el = frames.children;
|
||||
var to_update_text = [];
|
||||
for (var i = 0; i < el.length; i++) {
|
||||
var e = el[i];
|
||||
var a = find_child(e, "rect").attributes;
|
||||
var ex = parseInt(a["fg:x"].value);
|
||||
var ew = parseInt(a["fg:w"].value);
|
||||
// Is it an ancestor
|
||||
if (!inverted) {
|
||||
var upstack = parseFloat(a.y.value) > ymin;
|
||||
} else {
|
||||
var upstack = parseFloat(a.y.value) < ymin;
|
||||
}
|
||||
if (upstack) {
|
||||
// Direct ancestor
|
||||
if (ex <= xmin && (ex+ew) >= xmax) {
|
||||
e.classList.add("parent");
|
||||
zoom_parent(e);
|
||||
to_update_text.push(e);
|
||||
}
|
||||
// not in current path
|
||||
else
|
||||
e.classList.add("hide");
|
||||
}
|
||||
// Children maybe
|
||||
else {
|
||||
// no common path
|
||||
if (ex < xmin || ex >= xmax) {
|
||||
e.classList.add("hide");
|
||||
}
|
||||
else {
|
||||
zoom_child(e, xmin, width);
|
||||
to_update_text.push(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
update_text_for_elements(to_update_text);
|
||||
}
|
||||
function unzoom() {
|
||||
unzoombtn.classList.add("hide");
|
||||
var el = frames.children;
|
||||
for(var i = 0; i < el.length; i++) {
|
||||
el[i].classList.remove("parent");
|
||||
el[i].classList.remove("hide");
|
||||
zoom_reset(el[i]);
|
||||
}
|
||||
update_text_for_elements(el);
|
||||
}
|
||||
// search
|
||||
function reset_search() {
|
||||
var el = document.querySelectorAll("#frames rect");
|
||||
for (var i = 0; i < el.length; i++) {
|
||||
orig_load(el[i], "fill")
|
||||
}
|
||||
var params = get_params();
|
||||
delete params.s;
|
||||
history.replaceState(null, null, parse_params(params));
|
||||
}
|
||||
function search_prompt() {
|
||||
if (!searching) {
|
||||
var term = prompt("Enter a search term (regexp " +
|
||||
"allowed, eg: ^ext4_)", "");
|
||||
if (term != null) {
|
||||
search(term)
|
||||
}
|
||||
} else {
|
||||
reset_search();
|
||||
searching = 0;
|
||||
searchbtn.classList.remove("show");
|
||||
searchbtn.firstChild.nodeValue = "Search"
|
||||
matchedtxt.classList.add("hide");
|
||||
matchedtxt.firstChild.nodeValue = ""
|
||||
}
|
||||
}
|
||||
function search(term) {
|
||||
var re = new RegExp(term);
|
||||
var el = frames.children;
|
||||
var matches = new Object();
|
||||
var maxwidth = 0;
|
||||
for (var i = 0; i < el.length; i++) {
|
||||
var e = el[i];
|
||||
// Skip over frames which are either not visible, or below the zoomed-to frame
|
||||
if (e.classList.contains("hide") || e.classList.contains("parent")) {
|
||||
continue;
|
||||
}
|
||||
var func = g_to_func(e);
|
||||
var rect = find_child(e, "rect");
|
||||
if (func == null || rect == null)
|
||||
continue;
|
||||
// Save max width. Only works as we have a root frame
|
||||
var w = parseInt(rect.attributes["fg:w"].value);
|
||||
if (w > maxwidth)
|
||||
maxwidth = w;
|
||||
if (func.match(re)) {
|
||||
// highlight
|
||||
var x = parseInt(rect.attributes["fg:x"].value);
|
||||
orig_save(rect, "fill");
|
||||
rect.attributes.fill.value = searchcolor;
|
||||
// remember matches
|
||||
if (matches[x] == undefined) {
|
||||
matches[x] = w;
|
||||
} else {
|
||||
if (w > matches[x]) {
|
||||
// overwrite with parent
|
||||
matches[x] = w;
|
||||
}
|
||||
}
|
||||
searching = 1;
|
||||
}
|
||||
}
|
||||
if (!searching)
|
||||
return;
|
||||
var params = get_params();
|
||||
params.s = term;
|
||||
history.replaceState(null, null, parse_params(params));
|
||||
|
||||
searchbtn.classList.add("show");
|
||||
searchbtn.firstChild.nodeValue = "Reset Search";
|
||||
// calculate percent matched, excluding vertical overlap
|
||||
var count = 0;
|
||||
var lastx = -1;
|
||||
var lastw = 0;
|
||||
var keys = Array();
|
||||
for (k in matches) {
|
||||
if (matches.hasOwnProperty(k))
|
||||
keys.push(k);
|
||||
}
|
||||
// sort the matched frames by their x location
|
||||
// ascending, then width descending
|
||||
keys.sort(function(a, b){
|
||||
return a - b;
|
||||
});
|
||||
// Step through frames saving only the biggest bottom-up frames
|
||||
// thanks to the sort order. This relies on the tree property
|
||||
// where children are always smaller than their parents.
|
||||
for (var k in keys) {
|
||||
var x = parseInt(keys[k]);
|
||||
var w = matches[keys[k]];
|
||||
if (x >= lastx + lastw) {
|
||||
count += w;
|
||||
lastx = x;
|
||||
lastw = w;
|
||||
}
|
||||
}
|
||||
// display matched percent
|
||||
matchedtxt.classList.remove("hide");
|
||||
var pct = 100 * count / maxwidth;
|
||||
if (pct != 100) pct = pct.toFixed(1);
|
||||
matchedtxt.firstChild.nodeValue = "Matched: " + pct + "%";
|
||||
}
|
||||
function format_percent(n) {
|
||||
return n.toFixed(4) + "%";
|
||||
}
|
||||
]]></script><rect x="0" y="0" width="100%" height="918" fill="url(#background)"/><text id="title" fill="rgb(0,0,0)" x="50.0000%" y="24.00">Flame Graph</text><text id="details" fill="rgb(0,0,0)" x="10" y="901.00"> </text><text id="unzoom" class="hide" fill="rgb(0,0,0)" x="10" y="24.00">Reset Zoom</text><text id="search" fill="rgb(0,0,0)" x="1190" y="24.00">Search</text><text id="matched" fill="rgb(0,0,0)" x="1190" y="901.00"> </text><svg id="frames" x="10" width="1180" total_samples="155688399"><g><title><alloc::string::String as core::fmt::Write>::write_str (1,932,904 samples, 1.24%)</title><rect x="0.0000%" y="837" width="1.2415%" height="15" fill="rgb(227,0,7)" fg:x="0" fg:w="1932904"/><text x="0.2500%" y="847.50"></text></g><g><title><console::utils::StyledObject<D> as core::fmt::Display>::fmt (501,504 samples, 0.32%)</title><rect x="1.2415%" y="837" width="0.3221%" height="15" fill="rgb(217,0,24)" fg:x="1932904" fg:w="501504"/><text x="1.4915%" y="847.50"></text></g><g><title><diesel::sqlite::connection::SqliteConnection as diesel::connection::Connection>::establish (1,202,489 samples, 0.77%)</title><rect x="1.5636%" y="837" width="0.7724%" height="15" fill="rgb(221,193,54)" fg:x="2434408" fg:w="1202489"/><text x="1.8136%" y="847.50"></text></g><g><title><diesel::sqlite::query_builder::SqliteQueryBuilder as diesel::query_builder::QueryBuilder<diesel::sqlite::backend::Sqlite>>::push_sql (1,045,240 samples, 0.67%)</title><rect x="2.3360%" y="837" width="0.6714%" height="15" fill="rgb(248,212,6)" fg:x="3636897" fg:w="1045240"/><text x="2.5860%" y="847.50"></text></g><g><title>[db] (652,185 samples, 0.42%)</title><rect x="3.0074%" y="837" width="0.4189%" height="15" fill="rgb(208,68,35)" fg:x="4682137" fg:w="652185"/><text x="3.2574%" y="847.50"></text></g><g><title>std::io::Write::write_all (652,185 samples, 0.42%)</title><rect x="3.0074%" y="821" width="0.4189%" height="15" fill="rgb(232,128,0)" fg:x="4682137" fg:w="652185"/><text x="3.2574%" y="831.50"></text></g><g><title><serde_json::read::StrRead as serde_json::read::Read>::parse_str (880,273 samples, 0.57%)</title><rect x="3.4263%" y="821" width="0.5654%" height="15" fill="rgb(207,160,47)" fg:x="5334322" fg:w="880273"/><text x="3.6763%" y="831.50"></text></g><g><title>[unknown] (1,181,670 samples, 0.76%)</title><rect x="3.9917%" y="821" width="0.7590%" height="15" fill="rgb(228,23,34)" fg:x="6214595" fg:w="1181670"/><text x="4.2417%" y="831.50"></text></g><g><title>serde_json::value::de::<impl serde_core::de::Deserialize for serde_json::value::Value>::deserialize (1,181,670 samples, 0.76%)</title><rect x="3.9917%" y="805" width="0.7590%" height="15" fill="rgb(218,30,26)" fg:x="6214595" fg:w="1181670"/><text x="4.2417%" y="815.50"></text></g><g><title>[unknown] (2,563,447 samples, 1.65%)</title><rect x="3.4263%" y="837" width="1.6465%" height="15" fill="rgb(220,122,19)" fg:x="5334322" fg:w="2563447"/><text x="3.6763%" y="847.50"></text></g><g><title>indicatif::state::ProgressState::eta (501,504 samples, 0.32%)</title><rect x="4.7507%" y="821" width="0.3221%" height="15" fill="rgb(250,228,42)" fg:x="7396265" fg:w="501504"/><text x="5.0007%" y="831.50"></text></g><g><title><diesel::sqlite::connection::SqliteConnection as diesel::connection::SimpleConnection>::batch_execute (829,243 samples, 0.53%)</title><rect x="5.0728%" y="645" width="0.5326%" height="15" fill="rgb(240,193,28)" fg:x="7897769" fg:w="829243"/><text x="5.3228%" y="655.50"></text></g><g><title>diesel::sqlite::connection::raw::RawConnection::exec (829,243 samples, 0.53%)</title><rect x="5.0728%" y="629" width="0.5326%" height="15" fill="rgb(216,20,37)" fg:x="7897769" fg:w="829243"/><text x="5.3228%" y="639.50"></text></g><g><title>sqlite3_prepare_v2 (829,243 samples, 0.53%)</title><rect x="5.0728%" y="613" width="0.5326%" height="15" fill="rgb(206,188,39)" fg:x="7897769" fg:w="829243"/><text x="5.3228%" y="623.50"></text></g><g><title><indicatif::rayon::ProgressFolder<C> as rayon::iter::plumbing::Folder<T>>::consume (1,330,747 samples, 0.85%)</title><rect x="5.0728%" y="677" width="0.8548%" height="15" fill="rgb(217,207,13)" fg:x="7897769" fg:w="1330747"/><text x="5.3228%" y="687.50"></text></g><g><title>db::db::connect (1,330,747 samples, 0.85%)</title><rect x="5.0728%" y="661" width="0.8548%" height="15" fill="rgb(231,73,38)" fg:x="7897769" fg:w="1330747"/><text x="5.3228%" y="671.50"></text></g><g><title>core::result::unwrap_failed (501,504 samples, 0.32%)</title><rect x="5.6054%" y="645" width="0.3221%" height="15" fill="rgb(225,20,46)" fg:x="8727012" fg:w="501504"/><text x="5.8554%" y="655.50"></text></g><g><title>core::panicking::panic_fmt (501,504 samples, 0.32%)</title><rect x="5.6054%" y="629" width="0.3221%" height="15" fill="rgb(210,31,41)" fg:x="8727012" fg:w="501504"/><text x="5.8554%" y="639.50"></text></g><g><title>__rustc::rust_begin_unwind (501,504 samples, 0.32%)</title><rect x="5.6054%" y="613" widtLine truncated
|
||||
|
After Width: | Height: | Size: 113 KiB |
+46
-4
@@ -1,12 +1,55 @@
|
||||
use diesel::prelude::*;
|
||||
use diesel::connection::SimpleConnection;
|
||||
use diesel::r2d2::{ConnectionManager, CustomizeConnection, Pool};
|
||||
use std::env;
|
||||
use crate::models::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Fuck ();
|
||||
|
||||
impl CustomizeConnection<SqliteConnection, diesel::r2d2::Error> for Fuck {
|
||||
fn on_acquire (&self, conn: &mut SqliteConnection) -> Result<
|
||||
(), diesel::r2d2::Error
|
||||
> {
|
||||
conn.batch_execute ("
|
||||
PRAGMA busy_timeout = 15000;
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
PRAGMA wal_autocheckpoint = 1000;
|
||||
PRAGMA wal_checkpoint(TRUNCATE);
|
||||
PRAGMA cache_size=-2000000
|
||||
").unwrap ();
|
||||
Ok (())
|
||||
}
|
||||
|
||||
fn on_release (&self, _conn: SqliteConnection) {}
|
||||
}
|
||||
|
||||
pub fn get_pool () -> Pool<ConnectionManager<SqliteConnection>> {
|
||||
let url = env::var ("DATABASE_URL")
|
||||
.expect ("DATABASE_URL must be set");
|
||||
let manager = ConnectionManager::<SqliteConnection>::new (url);
|
||||
Pool::builder ()
|
||||
.connection_customizer (Box::new (Fuck ()))
|
||||
.test_on_check_out (true)
|
||||
.build (manager)
|
||||
.expect ("Could not build connection pool")
|
||||
}
|
||||
|
||||
pub fn connect () -> SqliteConnection {
|
||||
let database_url = env::var ("DATABASE_URL")
|
||||
.expect ("DATABASE_URL must be set");
|
||||
SqliteConnection::establish (&database_url)
|
||||
.unwrap_or_else (|_| panic! ("Error connecting to {}", database_url))
|
||||
let mut conn = SqliteConnection::establish (&database_url)
|
||||
.unwrap_or_else (|_| panic! ("Error connecting to {}", database_url));
|
||||
conn.batch_execute ("
|
||||
PRAGMA journal_mode = WAL; -- better write-concurrency
|
||||
PRAGMA synchronous = NORMAL; -- fsync only in critical moments
|
||||
PRAGMA wal_autocheckpoint = 1000; -- write WAL changes back every 1000 pages, for an in average 1MB WAL file. May affect readers if number is increased
|
||||
PRAGMA wal_checkpoint(TRUNCATE); -- free some space by truncating possibly massive WAL files from the last run.
|
||||
PRAGMA busy_timeout = 5000; -- sleep if the database is busy
|
||||
PRAGMA cache_size=-2000000
|
||||
").unwrap ();
|
||||
conn
|
||||
}
|
||||
|
||||
pub fn get_word (w : &str) -> Vec<Word> {
|
||||
@@ -39,7 +82,6 @@ pub fn insert_word (
|
||||
};
|
||||
diesel::insert_into (words::table)
|
||||
.values (new_word)
|
||||
.execute (conn)
|
||||
.expect ("error saving new word");
|
||||
.execute (conn).ok ()?;
|
||||
Some (())
|
||||
}
|
||||
+11
-8
@@ -4,10 +4,12 @@ pub mod db;
|
||||
|
||||
use flate2::read::GzDecoder;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use indicatif::ParallelProgressIterator;
|
||||
use serde_json::Value;
|
||||
use wiktionary_schema::en::WordData;
|
||||
use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf};
|
||||
use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
|
||||
use clap::Parser;
|
||||
use rayon::prelude::*;
|
||||
|
||||
/// Build an SQLite database from a Wiktextract JSONLines dump.
|
||||
#[derive(Parser, Debug)]
|
||||
@@ -40,18 +42,19 @@ fn main() -> Result<(), String> {
|
||||
"{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
|
||||
) .unwrap()
|
||||
);
|
||||
let conn = &mut db::connect ();
|
||||
let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
|
||||
.par_bridge ()
|
||||
.progress_with (bar);
|
||||
let pool = db::get_pool ();
|
||||
|
||||
for i in d.lines () {
|
||||
lines.for_each (|line| {
|
||||
let conn = &mut pool.clone ().get ().unwrap ();
|
||||
if let Ok (word) = serde_json::from_str::<Value> (
|
||||
&i.expect ("entry")
|
||||
&line
|
||||
) {
|
||||
bar.println (db::entry_as_word (&word).map_or ("", |x| x));
|
||||
db::insert_word (conn, word);
|
||||
bar.inc (1);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
bar.finish ();
|
||||
Ok (())
|
||||
}
|
||||
Binary file not shown.
Reference in new issue
Block a user