pub mod models;
pub mod schema;
pub mod db;

use diesel::SqliteConnection;
use diesel::r2d2::{ConnectionManager, Pool, PooledConnection};
use flate2::read::GzDecoder;
use indicatif::{ProgressBar, ProgressStyle};
use indicatif::ParallelProgressIterator;
use serde_json::Value;
use wiktionary_schema::en::WordData;
use std::{fs::File, io::{self, BufRead, BufReader}, path::PathBuf, sync::RwLock};
use clap::Parser;
use rayon::prelude::*;

/// Build an SQLite database from a Wiktextract JSONLines dump.
#[derive(Parser, Debug)]
#[command(name = "make-db", version, about, long_about = None)]
struct Args {
    /// Input file
    #[clap(long, value_parser)]
    wiktextract_dump : PathBuf,

    /// Output file
    #[clap(long, value_parser)]
    database : Option<PathBuf>,
}

fn open_wiktextract_dump (path : PathBuf) -> Result<
    BufReader<GzDecoder<File>>, io::Error
> {
    Ok (BufReader::new (GzDecoder::new (File::open (path)?)))
}

#[allow(nonstandard_style)]
const wiktextract_dump_lines : u64 = 10736399;

fn do_line (
    conn: &mut PooledConnection<ConnectionManager<SqliteConnection>>,
    retries : &RwLock<Vec<String>>,
    line : String
) {
    if let Ok (word) = serde_json::from_str::<Value> (&line) {
        if let None = db::insert_word (conn, word) {
            retries.write ().unwrap ().push (line);
        }
    }
}

fn main() -> Result<(), String> {
    let args = Args::parse ();
    let d = open_wiktextract_dump (args.wiktextract_dump).unwrap ();
    let bar = ProgressBar::new (wiktextract_dump_lines);
    bar.set_style (
        ProgressStyle::with_template(
            "{elapsed} / ETA {eta} / rate {per_sec} {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}"
        ) .unwrap()
    );
    let lines = d.lines ().map (|l| l.unwrap ().to_owned ())
        .par_bridge ()
        .progress_with (bar);
    let pool = db::get_pool ();
    let retries = RwLock::new (vec! []);

    lines.for_each (|line| {
        let conn = &mut pool.clone ().get ().unwrap ();
        do_line (conn, &retries, line)
    });

    println! ("do retries now");
    let conn = &mut pool.clone ().get ().unwrap ();
    for line in retries.read ().unwrap ().iter () {
        println! ("retrying");
        if let Ok (word) = serde_json::from_str::<Value> (&line) {
            db::insert_word (conn, word);
        }
    }

    Ok (())
}