Saving non-working state with temporary readme

.gitignore

@@ -1,3 +1,5 @@
 /target
-/idx
+/idx*
+/.err
 **/*.rs.bk
+/html_emails

Cargo.toml

@@ -7,24 +7,30 @@ edition = "2018"
 [dependencies]
 #maildir = { git = "https://github.com/lewisdiamond/maildir.git" }
 maildir = { path = "/home/ldiamond/dev/maildir/" }
-#html2text = "0.1.8"
+html2text = "0.1.13"
-html2text = { git = "https://github.com/lewisdiamond/rust-html2text.git"}
+#html2text = { git = "https://github.com/lewisdiamond/rust-html2text.git"}
-mailparse = "0.6.5"
+mailparse = "0.13.0"
-rayon = "1.0.3"
+rayon = "1.3.1"
-tantivy = "0.8.1"
+tantivy = "0.12.0"
 tempdir = "0.3.7"
-serde_json = "1.0.33"
+serde_json = "1.0.57"
-structopt = "0.2.14"
+serde = { version = "1.0.114", features = ["derive"] }
-shellexpand = "1.0"
+structopt = "0.3.15"
-log = { version = "0.4", features = ["max_level_debug", "release_max_level_warn"] } 
+shellexpand = "2.0.0"
-pretty_env_logger = "0.3"
+log = { version = "0.4.11", features = ["max_level_debug", "release_max_level_warn"] } 
-pbr = "1.0.1"
+pretty_env_logger = "0.4.0"
-num_cpus = "1.9.0"
+pbr = "1.0.3"
-sys-info = "0.5.6"
+num_cpus = "1.13.0"
-tui = "0.4.0"
+sys-info = "0.7.0"
-termion = "1.5.1"
+tui = "0.10.0"
-chrono = "0.4.6"
+termion = "1.5.5"
-sha2 = "0.8.0"
+chrono = "0.4.13"
-html5ever = "*"
+sha2 = "0.9.1"
-rocksdb = "0.12.0"
+html5ever = "0.25.1"
-serde = { version = "1.0.89", features = ["derive"]}
+rocksdb = { path = "../rust-rocksdb/" }
+jemallocator = "0.3.2"
+#maildir = "0.4.2"
+select = "0.5.0"
+
+[dev-dependencies]
+rand = "0.7.3"

README.md

@@ -0,0 +1,8 @@
+# This project is not currently in a working state. Contributions are welcome.
+
+RMS is an email system that allows you to easily search, tag, list and read your
+emails from the command line.
+
+It is similar to NotMuch but much faster and provides more ways to access your
+emails.
+

src/bin/readmail.rs

@@ -1,41 +1,33 @@
-use log::{debug, error};
+use log::error;
-use rms::message::{get_id, Message};
+use rms::message::Message;
-use rms::readmail::{extract_body, html2text};
+use rms::readmail::html2text;
 mod readmail_cmd;
-use mailparse::*;
 use readmail_cmd::source;
 use rms::message::Mime;
 use std::io::BufRead;
 
 fn main() {
-    let src: Box<BufRead> = source();
+    let src: Box<dyn BufRead> = source();
     let b_msg_rslt = src.split(3);
     for m in b_msg_rslt {
         match m {
             Ok(buf) => {
-                let hash = get_id(&buf);
+                let message = Message::from_data(buf);
-                if let Ok(mut msg) = parse_mail(buf.as_slice()) {
+                match message {
-                    let message = Message::from_parsedmail(&mut msg, hash);
+                    Ok(message) => {
-                    match message {
+                        println!("From: {}", message.from);
-                        Ok(message) => {
+                        println!("To: {}", message.recipients.join(", "));
-                            println!("From: {}", message.from);
+                        println!("Subject: {}", message.subject);
-                            println!("To: {}", message.recipients.join(", "));
+                        for b in message.body {
-                            println!("Subject: {}", message.subject);
+                            println!("Body Mime: {}", b.mime.as_str());
-                            let body = extract_body(&mut msg, false);
+                            match b.mime {
-                            for b in body {
+                                Mime::PlainText => println!("\n\n{}", b.value),
-                                println!("Body Mime: {}", b.mime.as_str());
+                                Mime::Html => println!("\n\n{}", html2text(&b.value)),
-                                match b.mime {
+                                _ => println!("Unknown mime type"),
-                                    Mime::PlainText => println!("\n\n{}", b.value),
-                                    Mime::Html => println!("\n\n{}", html2text(&b.value)),
-                                    _ => println!("Unknown mime type"),
-                                }
                             }
                         }
-                        Err(e) => error!("Failed to make sense of the message"),
                     }
-                } else {
+                    Err(_e) => error!("Failed to make sense of the message"),
-                    error!("Failed to parse the file");
-                    ::std::process::exit(1);
                 }
             }
             Err(e) => {

src/bin/readmail_cmd/mod.rs

@@ -18,6 +18,9 @@ pub fn source() -> Box<BufRead> {
         .args(&[Arg::from_usage(
             "[input] 'Read from a file, or stdin if omitted'",
         )])
+        .args(&[Arg::from_usage(
+            "[-o destination] 'Save attachment to destination'",
+        )])
         .get_matches();
 
     match matches.value_of("input") {

src/bin/rms.rs

@@ -1,8 +1,10 @@
 use log::{error, info, trace};
 use rms::cmd::{opts, Command, OutputType};
-use rms::stores::{MessageStoreBuilder, Searchers, Storages};
+use rms::stores::{IMessageStore, MessageStoreBuilder, Searchers, Storages};
 use rms::terminal;
 use std::collections::HashSet;
+use std::io::{self, Write};
+use std::time::Instant;
 
 fn main() {
     pretty_env_logger::init();
@@ -54,6 +56,7 @@ fn main() {
             //message_store.index_mails(full);
         }
         Command::Search { term, output, num } => {
+            let now = Instant::now();
             let message_store = MessageStoreBuilder::new()
                 .storage(Storages::Tantivy(index_dir_path.clone()))
                 .searcher(Searchers::Tantivy(index_dir_path.clone()))
@@ -72,6 +75,12 @@ fn main() {
                         OutputType::Full => {
                             println!("{:?}", results);
                         }
+                        OutputType::Raw => {
+                            let mut out = io::stdout();
+                            for result in results {
+                                out.write_all(result.original.as_ref()).unwrap();
+                            }
+                        }
                     }
                 }
                 Err(e) => error!("{}", e),
@@ -99,29 +108,64 @@ fn main() {
 
             match message_store {
                 Ok(store) => {
-                    let result = store.get_message(id).ok().unwrap();
+                    let result = store.get_message(id);
-                    match output {
+                    match result {
-                        OutputType::Short => {
+                        Ok(Some(good_msg)) => match output {
-                            println!("{:?} | {}", result.id, result.subject);
+                            OutputType::Short => {
-                        }
+                                println!("{} | {}", good_msg.id, good_msg.subject);
-                        OutputType::Full => {
+                            }
-                            println!("{:?}", result);
+                            OutputType::Raw => {
-                        }
+                                io::stdout().write_all(good_msg.original.as_ref()).unwrap();
+                            }
+                            OutputType::Full => {
+                                println!("From: {}", good_msg.from);
+                                println!(
+                                    "To: {}",
+                                    good_msg
+                                        .recipients
+                                        .get(0)
+                                        .unwrap_or(&String::from("Unknown"))
+                                );
+                                println!("Subject: {}", good_msg.subject);
+                                println!(
+                                    "{}",
+                                    good_msg
+                                        .body
+                                        .first()
+                                        .map(|b| b.value.clone())
+                                        .unwrap_or(String::from("No body"))
+                                );
+                            }
+                        },
+                        Ok(None) => error!("Message not found"),
+                        Err(e) => error!("ERROR {}", e),
                     }
                 }
-                Err(e) => error!("{}", e),
+                Err(e) => error!("Store isn't right... {}", e),
             }
         }
         Command::Interactive {} => {
             terminal::start(index_dir_path).unwrap();
         }
         Command::Latest { num: _num } => {
-            let _message_store = MessageStoreBuilder::new().build(); //maildir_path[0].clone(), index_dir_path);
+            let message_store = MessageStoreBuilder::new()
-                                                                     //let searcher = Searcher::new(index_dir_path);
+                .storage(Storages::Tantivy(index_dir_path.clone()))
-                                                                     //let stuff = searcher.latest(num, None);
+                .searcher(Searchers::Tantivy(index_dir_path.clone()))
-                                                                     //for s in stuff {
+                .build();
-                                                                     //    println!("{}", s.date);
+            match message_store {
-                                                                     //}
+                Ok(store) => {
+                    let page = store.get_messages_page(0, _num);
+                    match page {
+                        Ok(msgs) => {
+                            for m in msgs {
+                                println!("{}", m.id);
+                            }
+                        }
+                        Err(e) => println!("Could not read messages, {}", e),
+                    }
+                }
+                Err(e) => println!("Could not load the index, {}", e),
+            }
         }
         Command::Tag { id, tags } => {
             let message_store = MessageStoreBuilder::new()

src/cmd/mod.rs

@@ -18,6 +18,7 @@ pub fn expand_path(input: &OsStr) -> PathBuf {
 pub enum OutputType {
     Short,
     Full,
+    Raw,
 }
 
 #[derive(Debug)]
@@ -49,6 +50,7 @@ impl FromStr for OutputType {
         match input.to_lowercase().as_str() {
             "short" => Ok(OutputType::Short),
             "full" => Ok(OutputType::Full),
+            "raw" => Ok(OutputType::Raw),
             _ => Err(OutputTypeError::UnknownTypeError),
         }
     }
@@ -64,7 +66,7 @@ impl FromStr for OutputType {
 )]
 pub struct Opt {
     #[structopt(
-        parse(from_os_str = "expand_path"),
+        parse(from_os_str = expand_path),
         short,
         long,
         env = "RMS_CONFIG_PATH",
@@ -73,7 +75,7 @@ pub struct Opt {
     pub config: PathBuf,
 
     #[structopt(
-        parse(from_os_str = "expand_path"),
+        parse(from_os_str = expand_path),
         short,
         long,
         env = "RMS_INDEX_DIR_PATH"
@@ -89,7 +91,7 @@ pub enum Command {
     #[structopt(name = "index", rename_all = "kebab-case")]
     Index {
         #[structopt(
-            parse(from_os_str = "expand_path"),
+            parse(from_os_str = expand_path),
             short,
             long,
             required = true,
@@ -117,11 +119,10 @@ pub enum Command {
 
     #[structopt(rename_all = "kebab-case")]
     Get {
-        #[structopt(short, long)]
-        id: String,
-
         #[structopt(short, long, default_value = "short")]
         output: OutputType,
+
+        id: String,
     },
 
     #[structopt(rename_all = "kebab-case")]

src/lib.rs

@@ -6,9 +6,7 @@ extern crate num_cpus;
 extern crate pbr;
 extern crate pretty_env_logger;
 extern crate rayon;
-#[macro_use]
 extern crate serde;
-#[macro_use]
 extern crate serde_json;
 extern crate shellexpand;
 extern crate structopt;
@@ -22,3 +20,9 @@ pub mod message;
 pub mod readmail;
 pub mod stores;
 pub mod terminal;
+
+extern crate jemallocator;
+#[cfg(test)]
+extern crate rand;
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

src/message/mod.rs

@@ -1,16 +1,16 @@
 use crate::readmail;
 use crate::readmail::html2text;
 use chrono::prelude::*;
-use html2text::from_read;
+use maildir::{MailEntry, ParsedMailEntry};
-use log::{debug, error};
+use mailparse::{dateparse, parse_mail, ParsedMail};
-use maildir::MailEntry;
+use serde::{Deserialize, Serialize};
-use mailparse::{dateparse, ParsedMail};
 use sha2::{Digest, Sha512};
 use std::collections::HashSet;
 use std::convert::AsRef;
+use std::path::{Path, PathBuf};
 use std::string::ToString;
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Mime {
     PlainText,
     Html,
@@ -39,7 +39,7 @@ impl Mime {
         }
     }
 }
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Body {
     pub mime: Mime,
     pub value: String,
@@ -57,7 +57,7 @@ impl Body {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Message {
     pub id: String,
     pub body: Vec<Body>,
@@ -65,10 +65,18 @@ pub struct Message {
     pub from: String,
     pub recipients: Vec<String>,
     pub date: u64,
-    pub original: Option<String>,
+    pub original: Vec<u8>,
     pub tags: HashSet<String>,
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct ShortMessage {
+    pub id: String,
+    pub subject: String,
+    pub from: String,
+    pub date: u64,
+}
+
 pub struct MessageBuilder {
     body: Option<String>,
     subject: Option<String>,
@@ -76,12 +84,19 @@ pub struct MessageBuilder {
     recipients: Option<Vec<String>>,
     date: Option<u64>,
     id: Option<String>,
+    original: Option<Vec<u8>>,
 }
 
 pub fn get_id(data: &Vec<u8>) -> String {
-    let mut hasher = Sha512::default();
+    format!("{:x}", Sha512::digest(data))
-    hasher.input(data);
+}
-    format!("{:x}", hasher.result())
+
+impl ToString for ShortMessage {
+    fn to_string(&self) -> String {
+        let dt = Local.timestamp(self.date as i64, 0);
+        let dstr = dt.format("%a %b %e %T %Y").to_string();
+        format!("{}: [{}] {}", dstr, self.from, self.subject.as_str())
+    }
 }
 
 impl ToString for Message {
@@ -119,8 +134,20 @@ pub struct MessageError {
     pub message: String,
 }
 
+impl MessageError {
+    pub fn from(msg: &str) -> Self {
+        MessageError {
+            message: String::from(msg),
+        }
+    }
+}
+
 impl Message {
-    pub fn from_parsedmail(mut msg: &mut ParsedMail, id: String) -> Result<Self, MessageError> {
+    pub fn from_parsedmail(
+        msg: &ParsedMail,
+        id: String,
+        original: Vec<u8>,
+    ) -> Result<Self, MessageError> {
         let headers = &msg.headers;
         let mut subject: String = "".to_string();
         let mut from: String = "".to_string();
@@ -128,35 +155,29 @@ impl Message {
         let default_date = 0;
         let mut date = default_date;
         for h in headers {
-            if let Ok(s) = h.get_key() {
+            let key = h.get_key();
-                match s.as_ref() {
+            match key.as_ref() {
-                    "Subject" => subject = h.get_value().unwrap_or("".to_string()),
+                "Subject" => subject = h.get_value(),
-                    "From" => from = h.get_value().unwrap_or("".to_string()),
+                "From" => from = h.get_value(),
-                    "To" => recipients.push(h.get_value().unwrap_or("".to_string())),
+                "To" => recipients.push(h.get_value()),
-                    "cc" => recipients.push(h.get_value().unwrap_or("".to_string())),
+                "cc" => recipients.push(h.get_value()),
-                    "bcc" => recipients.push(h.get_value().unwrap_or("".to_string())),
+                "bcc" => recipients.push(h.get_value()),
-                    "Received" | "Date" => {
+                "Received" | "Date" => {
-                        if date == default_date {
+                    if date == default_date {
-                            let date_str = h.get_value();
+                        let date_str = h.get_value();
-                            match date_str {
+                        for ts in date_str.rsplit(';') {
-                                Ok(date_str) => {
+                            date = match dateparse(ts) {
-                                    for ts in date_str.rsplit(';') {
+                                Ok(d) => d,
-                                        date = match dateparse(ts) {
+                                Err(_) => default_date,
-                                            Ok(d) => d,
+                            };
-                                            Err(_) => default_date,
+                            break;
-                                        };
-                                        break;
-                                    }
-                                }
-                                Err(_) => (),
-                            }
                         }
                     }
-                    _ => {}
                 }
+                _ => {}
             }
         }
-        let bodies = readmail::extract_body(&mut msg, false);
+        let bodies = readmail::extract_body(&msg, false);
         Ok(Message {
             body: bodies,
             from,
@@ -164,21 +185,29 @@ impl Message {
             recipients,
             date: date as u64,
             id,
-            original: None,
+            original,
             tags: HashSet::new(),
         })
     }
-    pub fn from_mailentry(mailentry: &mut MailEntry) -> Result<Self, MessageError> {
+    pub fn from_data(data: Vec<u8>) -> Result<Self, MessageError> {
-        let hash = get_id(mailentry.data().expect("Unable to read the actual data"));
+        let id = get_id(data.as_ref());
+        let parsed_mail = parse_mail(data.as_slice()).map_err(|_| MessageError {
+            message: String::from("Unable to parse email data"),
+        })?;
+        Self::from_parsedmail(&parsed_mail, id, data.clone())
+    }
+    pub fn from_mailentry(mailentry: MailEntry) -> Result<Self, MessageError> {
+        let id = mailentry.id();
+        mailentry.read_data().map_err(|e| MessageError {
+            message: format!("Failed to parse email id {}", id),
+        })?;
+        let data = mailentry.data().ok_or(MessageError {
+            message: format!("Mail {} could not read data", id),
+        })?;
         match mailentry.parsed() {
-            Ok(mut msg) => Self::from_parsedmail(&mut msg, hash),
+            Ok(parsed) => Self::from_parsedmail(&parsed, String::from(id), data.clone()),
             Err(e) => Err(MessageError {
-                message: format!(
+                message: format!("Failed to parse email id {}", id),
-                    "Failed on {}:{} -- MailEntryError: {}",
-                    mailentry.id(),
-                    hash,
-                    e
-                ),
             }),
         }
     }
@@ -231,6 +260,10 @@ impl MessageBuilder {
         self.recipients = Some(recipients.split(",").map(|s| String::from(s)).collect());
         self
     }
+    fn original(mut self, original: Vec<u8>) -> Self {
+        self.original = Some(original);
+        self
+    }
     fn build(self) -> Message {
         let msg = "Missing field for Message";
 
@@ -244,7 +277,7 @@ impl MessageBuilder {
             subject: self.subject.expect(msg),
             recipients: self.recipients.expect(msg),
             date: self.date.expect(msg),
-            original: None,
+            original: self.original.expect(msg),
             tags: HashSet::new(),
         }
     }

src/readmail/mod.rs

@@ -1,13 +1,10 @@
-use crate::message::{Body, Mime};
+extern crate select;
-use html5ever::rcdom::{Handle, Node, NodeData, RcDom};
+use crate::message::{get_id, Body, Message, Mime};
-use html5ever::serialize::{serialize, SerializeOpts};
+use log::debug;
-use html5ever::tendril::TendrilSink;
-use html5ever::tree_builder::TreeBuilderOpts;
-use html5ever::{local_name, parse_document, ParseOpts};
-use log::{debug, error};
 use mailparse::*;
+use select::document::Document;
+use select::predicate::Text;
 use std::cmp::Ordering;
-use std::time::{Duration, Instant};
 
 fn cmp_body(x: &Body, y: &Body, prefer: &Mime) -> Ordering {
     if x.mime == y.mime {
@@ -20,20 +17,23 @@ fn cmp_body(x: &Body, y: &Body, prefer: &Mime) -> Ordering {
     }
 }
 
-pub fn extract_body(msg: &mut ParsedMail, prefer_html: bool) -> Vec<Body> {
+pub fn extract_body(msg: &ParsedMail, prefer_html: bool) -> Vec<Body> {
-    let mut raw_body = None;
     let prefered_mime = if prefer_html {
         Mime::Html
     } else {
         Mime::PlainText
     };
-    if let Ok(text) = msg.get_body() {
+    let text = msg
-        let mime = Mime::from_str(&msg.ctype.mimetype);
+        .get_body()
-        raw_body = Some(Body::new(mime, String::from(text)));
+        .unwrap_or(msg.get_body_raw().map_or(String::from(""), |x| {
-    };
+            String::from_utf8(x).unwrap_or(String::from(""))
+        }));
+    let mime = Mime::from_str(&msg.ctype.mimetype);
+    let raw_body = Some(Body::new(mime, text));
+
     let mut bodies = msg
         .subparts
-        .iter_mut()
+        .iter()
         .map(|mut s| {
             let mime = Mime::from_str(&s.ctype.mimetype);
             match mime {
@@ -53,52 +53,24 @@ pub fn extract_body(msg: &mut ParsedMail, prefer_html: bool) -> Vec<Body> {
         bodies.push(raw_body.expect("COULD NOT UNWRAP RAW_BODY"));
     }
     bodies.sort_unstable_by(|x, y| cmp_body(x, y, &prefered_mime));
+    if bodies.len() == 0 {
+        println!(
+            "No body for message: {}",
+            msg.headers
+                .iter()
+                .map(|x| format!("{}:{}", x.get_key(), x.get_value()))
+                .collect::<Vec<String>>()
+                .join("\n")
+        );
+    }
     bodies
 }
 
 pub fn html2text(text: &str) -> String {
-    let opts = ParseOpts {
+    let document = Document::from(text);
-        tree_builder: TreeBuilderOpts {
+    let text_nodes = document
-            drop_doctype: true,
+        .find(Text)
-            ..Default::default()
+        .map(|x| x.text())
-        },
+        .collect::<Vec<String>>();
-        ..Default::default()
+    return text_nodes.join("\n\n");
-    };
-    let dom = parse_document(RcDom::default(), opts)
-        .from_utf8()
-        .read_from(&mut text.as_bytes())
-        .expect("COULD NOT UNWRAP DOM");
-    let document_children = dom.document.children.borrow();
-    let html = document_children.get(0).expect("COULD NOT UNWRAP HTML");
-    let body_rc = html.children.borrow();
-    let body = body_rc
-        .iter()
-        .filter(|n| match n.data {
-            NodeData::Element { ref name, .. } => name.local == local_name!("body"),
-            _ => false,
-        })
-        .next();
-
-    let ret = match body {
-        Some(b) => render_tag(&b)
-            .into_iter()
-            .filter(|s| s != "")
-            .map(|s| s.trim().to_string())
-            .collect::<Vec<String>>()
-            .join("\n"),
-        None => "".to_string(),
-    };
-    ret
-}
-
-pub fn render_tag(node: &Handle) -> Vec<String> {
-    let mut ret = vec![];
-    match node.data {
-        NodeData::Text { ref contents } => ret.push(contents.borrow().trim().to_string()),
-        _ => {}
-    };
-    for child in node.children.borrow().iter() {
-        ret.append(&mut render_tag(child));
-    }
-    ret
 }

src/stores/_impl/rocksdb.rs

@@ -1,27 +1,38 @@
-use crate::message::{Body, Message, Mime};
+use crate::message::{Message, ShortMessage};
-use crate::stores::{IMessageSearcher, IMessageStorage, MessageStoreError};
+use crate::stores::{IMessageStorage, MessageStoreError};
 use chrono::{DateTime, Utc};
-use log::{info, trace, warn};
+use rocksdb::{DBCompactionStyle, DBCompressionType};
 use rocksdb::{DBVector, Options, DB};
-use serde::{Deserialize, Serialize};
+use serde_json::Result as SResult;
-use std::cmp;
 use std::collections::HashSet;
-use std::path::PathBuf;
+use std::path::Path;
 use std::string::ToString;
-const BYTES_IN_MB: usize = 1024 * 1024;
 
-#[derive(Serialize, Deserialize)]
+type RocksDBMessage = Message;
-struct RocksDBMessage {
-    id: String,
-    body: String,
-    tag: HashSet<String>,
-}
 impl RocksDBMessage {
-    fn from(doc: DBVector) -> RocksDBMessage {
+    fn from_rocksdb(msg: DBVector) -> Result<RocksDBMessage, MessageStoreError> {
-        RocksDBMessage {
+        let msg_r = msg
-            id: "a".to_string(),
+            .to_utf8()
-            body: "b".to_string(),
+            .ok_or(Err(MessageStoreError::CouldNotGetMessage(
-            tag: HashSet::new(),
+                "Message is malformed in some way".to_string(),
+            )));
+
+        match msg_r {
+            Ok(msg) => serde_json::from_str(msg).map_err(|e| {
+                MessageStoreError::CouldNotGetMessage("Unable to parse the value".to_string())
+            }),
+            Err(e) => e,
+        }
+    }
+    fn to_rocksdb(&self) -> Result<(String, Vec<u8>), MessageStoreError> {
+        let id = self.id.clone();
+        let msg = serde_json::to_string(&self);
+        match msg {
+            Ok(msg) => Ok((id, msg.into_bytes())),
+            Err(e) => Err(MessageStoreError::CouldNotConvertMessage(format!(
+                "Failed to convert message for rocksdb: {}",
+                e
+            ))),
         }
     }
 }
@@ -29,15 +40,36 @@ impl RocksDBMessage {
 pub struct RocksDBStore {
     db: DB,
 }
+
 impl IMessageStorage for RocksDBStore {
-    fn get_message(&self, id: String) -> Result<Message, MessageStoreError> {
+    fn add_message(&mut self, msg: &RocksDBMessage) -> Result<String, MessageStoreError> {
-        self.get_message(id.as_str())
+        let rocks_msg = msg.to_rocksdb();
-            .ok_or(MessageStoreError::MessageNotFound(
+        match rocks_msg {
-                "Unable to find message with that id".to_string(),
+            Ok((id, data)) => self
-            ))
+                .db
+                .put(id.clone().into_bytes(), data)
+                .map_err(|_| {
+                    MessageStoreError::CouldNotAddMessage("Failed to add message".to_string())
+                })
+                .map(|_| id),
+            Err(e) => Err(MessageStoreError::CouldNotAddMessage(format!(
+                "Failed to add message: {}",
+                e
+            ))),
+        }
     }
-    fn add_message(&mut self, msg: Message) -> Result<String, MessageStoreError> {
+    fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError> {
-        unimplemented!();
+        let m = self.db.get(id.into_bytes());
+        match m {
+            Ok(Some(message)) => Ok(Some(RocksDBMessage::from_rocksdb(message)?)),
+            Ok(None) => Err(MessageStoreError::CouldNotGetMessage(
+                "Message obtained was None".to_string(),
+            )),
+            Err(e) => Err(MessageStoreError::CouldNotGetMessage(format!(
+                "Could not get message due to : {}",
+                e
+            ))),
+        }
     }
     fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
         unimplemented!()
@@ -60,127 +92,105 @@ impl IMessageStorage for RocksDBStore {
         unimplemented!()
     }
 }
-impl IMessageSearcher for TantivyStore {
-    fn start_indexing_process(&mut self, num: usize) -> Result<(), MessageStoreError> {
-        if self.index_writer.is_none() {
-            let index_writer = self.get_index_writer(num)?;
-            self.index_writer = Some(index_writer);
-        }
-        Ok(())
-    }
-
-    fn finish_indexing_process(&mut self) -> Result<(), MessageStoreError> {
-        let writer = &mut self.index_writer;
-        match writer {
-            Some(writer) => match writer.commit() {
-                Ok(_) => Ok(()),
-                Err(e) => Err(MessageStoreError::CouldNotAddMessage(
-                    "Failed to commit to index".to_string(),
-                )),
-            },
-            None => Err(MessageStoreError::CouldNotAddMessage(
-                "Trying to commit index without an actual index".to_string(),
-            )),
-        }
-    }
-
-    fn add_message(
-        &mut self,
-        msg: Message,
-        parsed_body: String,
-    ) -> Result<String, MessageStoreError> {
-        self._add_message(msg, parsed_body)
-    }
-    fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError> {
-        Ok(self.fuzzy(query.as_str(), num))
-    }
-    fn search_by_date(
-        &self,
-        start: DateTime<Utc>,
-        end: DateTime<Utc>,
-    ) -> Result<Vec<Message>, MessageStoreError> {
-        Ok(vec![])
-    }
-    fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
-        Ok(())
-    }
-    fn tag_message_id(
-        &mut self,
-        id: String,
-        tags: HashSet<String>,
-    ) -> Result<usize, MessageStoreError> {
-        let message = self.get_message(id.as_str());
-        match message {
-            Some(mut message) => {
-                let now = Instant::now();
-                self.start_indexing_process(1)?;
-                println!("{}", now.elapsed().as_nanos());
-                self._delete_message(&message)?;
-                println!("{}", now.elapsed().as_nanos());
-                message.tags = tags;
-                let body = message.get_body().clone();
-                self._add_message(message, body.value)?;
-                println!("{}", now.elapsed().as_nanos());
-                self.finish_indexing_process()?;
-                println!("{}", now.elapsed().as_nanos());
-                Ok(1)
-            }
-            None => Err(MessageStoreError::MessageNotFound(
-                "Could not tag message because the message was not found".to_string(),
-            )),
-        }
-    }
-    fn tag_message(
-        &mut self,
-        msg: Message,
-        tags: HashSet<String>,
-    ) -> Result<usize, MessageStoreError> {
-        Ok(1)
-    }
-    fn get_messages_page(
-        &self,
-        start: Message,
-        num: usize,
-    ) -> Result<Vec<Message>, MessageStoreError> {
-        Ok(vec![])
-    }
-    fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
-        unimplemented!();
-    }
-}
 
 impl RocksDBStore {
-    pub fn new(path: PathBuf) -> Self {
+    pub fn new<P: AsRef<Path>>(path: P) -> Self {
-        unimplemented!()
+        let mut opts = Options::default();
+        opts.increase_parallelism(16);
+        opts.create_if_missing(true);
+        opts.set_compaction_style(DBCompactionStyle::Level);
+        opts.set_skip_stats_update_on_db_open(true);
+        opts.set_compression_type(DBCompressionType::Lz4);
+        opts.create_missing_column_families(true);
+        opts.set_use_direct_reads(true);
+        opts.set_allow_mmap_reads(true);
+        opts.set_allow_mmap_writes(true);
+        opts.set_max_open_files(2);
+        let db = DB::open_default(path).unwrap();
+        RocksDBStore { db }
     }
     fn _add_message(
         &mut self,
-        msg: Message,
+        _msg: &Message,
-        parsed_body: String,
+        _parsed_body: String,
     ) -> Result<String, MessageStoreError> {
         unimplemented!()
     }
-    fn _delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError> {
+    fn _delete_message(&mut self, _msg: &Message) -> Result<(), MessageStoreError> {
-        unimplemented!()
-    }
-    pub fn tag_doc(&self, doc: Document, tags: Vec<String>) -> Result<(), MessageStoreError> {
         unimplemented!()
     }
 
-    pub fn latest(&self, num: usize) -> Vec<Message> {
+    pub fn latest(&self, _num: usize) -> Vec<Message> {
         unimplemented!()
     }
-
+}
-    pub fn by_date(&self) {
+
-        unimplemented!();
+#[cfg(test)]
-    }
+mod test {
-    pub fn get_doc(&self, id: &str) -> Result<Document, tantivy::Error> {
+    use super::RocksDBStore;
-        unimplemented!();
+    use crate::message::{Body, Message, Mime};
-    }
+    use crate::stores::IMessageStorage;
-    pub fn get_message(&self, id: &str) -> Option<Message> {
+    use rand::distributions::Alphanumeric;
-        unimplemented!();
+    use rand::{thread_rng, Rng};
-    }
+    use rocksdb::{Options, DB};
-    pub fn search(&self, text: &str, num: usize) -> Vec<Message> {
+    use std::collections::HashSet;
-        unimplemented!();
+
-    }
+    struct StoreInit {
+        path: Option<String>,
+        store: RocksDBStore,
+    }
+    impl StoreInit {
+        fn new() -> StoreInit {
+            let rand_string: String = thread_rng().sample_iter(&Alphanumeric).take(5).collect();
+            let mut path = std::path::PathBuf::new();
+            path.push("./test_db/");
+            path.push(rand_string);
+            let newdb = RocksDBStore::new(&path);
+            StoreInit {
+                path: path.to_str().map(|s| s.to_string()),
+                store: newdb,
+            }
+        }
+    }
+    impl Drop for StoreInit {
+        fn drop(&mut self) {
+            let opts = Options::default();
+            let path = self.path.as_ref().unwrap();
+            DB::destroy(&opts, path);
+            std::fs::remove_dir_all(path);
+        }
+    }
+    #[test]
+    fn add_message() {
+        let store = &mut StoreInit::new().store;
+        let message = Message {
+            id: "some_id".to_string(),
+            from: "It's me, Mario!".to_string(),
+            body: vec![Body {
+                mime: Mime::PlainText,
+                value: "Test body".to_string(),
+            }],
+            subject: "test_subject".to_string(),
+            recipients: vec!["r1".to_string(), "r2".to_string()],
+            date: 4121251,
+            original: vec![0],
+            tags: vec!["tag1".to_string(), "tag2".to_string()]
+                .into_iter()
+                .collect::<HashSet<String>>(),
+        };
+        store.add_message(&message).ok().unwrap();
+        let retrieved = store
+            .get_message("some_id".to_string())
+            .ok()
+            .unwrap()
+            .unwrap();
+        assert_eq!(message, retrieved);
+    }
+    #[test]
+    fn test_rocksdb2() {
+        let store = &StoreInit::new().store;
+        store.db.put(b"key", b"value2").unwrap();
+        let get = store.db.get(b"key").ok().unwrap().unwrap();
+        assert_eq!("value2", get.to_utf8().unwrap());
+    }
 }

src/stores/_impl/tantivy.rs

@@ -1,13 +1,11 @@
 use crate::stores::{IMessageSearcher, IMessageStorage, MessageStoreError};
 use chrono::{DateTime, Utc};
-use log::{debug, error, info, trace, warn};
+use log::info;
 use std::cmp;
 use std::collections::HashSet;
 use std::fs;
 use std::panic;
-use std::sync::mpsc;
+use std::time::Instant;
-use std::time::{Duration, Instant};
-use std::time::{SystemTime, UNIX_EPOCH};
 
 use crate::message::{Body, Message, Mime};
 use std::path::PathBuf;
@@ -17,18 +15,22 @@ use tantivy::collector::{Count, TopDocs};
 use tantivy::directory::MmapDirectory;
 use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, RangeQuery, TermQuery};
 use tantivy::schema::*;
-use tantivy::DocAddress;
 const BYTES_IN_MB: usize = 1024 * 1024;
 
 pub type TantivyMessage = Message;
-impl TantivyMessage {
+
-    fn from(doc: Document, schema: &EmailSchema) -> TantivyMessage {
+pub trait TantivyFrom<T> {
-        let original = match doc.get_first(schema.original) {
+    fn from_tantivy(doc: Document, schema: &EmailSchema) -> T;
-            Some(t) => match t.text() {
+}
-                Some(t) => Some(String::from(t)),
+
-                None => None,
+impl TantivyFrom<TantivyMessage> for TantivyMessage {
-            },
+    fn from_tantivy(doc: Document, schema: &EmailSchema) -> TantivyMessage {
-            None => None,
+        let original: Result<Vec<u8>, _> = match doc
+            .get_first(schema.original)
+            .expect("Unable to get original message")
+        {
+            Value::Bytes(b) => Ok(b.clone()),
+            _ => Err("Missing original email from the index"),
         };
 
         let tags: HashSet<String> = doc
@@ -37,7 +39,6 @@ impl TantivyMessage {
             .filter_map(|s| s.text())
             .map(|s| String::from(s))
             .collect();
-
         TantivyMessage {
             id: doc
                 .get_first(schema.id)
@@ -45,12 +46,29 @@ impl TantivyMessage {
                 .text()
                 .expect("Message ID is always a string")
                 .to_string(),
+            from: String::from(
+                doc.get_first(schema.from)
+                    .expect("Message without from")
+                    .text()
+                    .expect("Message with non-text from"),
+            ),
             subject: String::from(
                 doc.get_first(schema.subject)
                     .expect("Message without subject")
                     .text()
                     .expect("Message with non-text subject"),
             ),
+            date: doc
+                .get_first(schema.date)
+                .map_or(0, |v: &tantivy::schema::Value| v.u64_value()),
+            recipients: doc
+                .get_first(schema.recipients)
+                .unwrap_or(&tantivy::schema::Value::Str(String::from("a")))
+                .text()
+                .expect("Message with non-text recipients")
+                .split(",")
+                .map(|s| String::from(s))
+                .collect(),
             body: vec![Body {
                 mime: Mime::PlainText,
                 value: String::from(
@@ -60,30 +78,14 @@ impl TantivyMessage {
                         .expect("Message with non-text body"),
                 ),
             }],
-            from: String::from(
+
-                doc.get_first(schema.from)
+            original: original.expect("Original was missing from the index"),
-                    .expect("Message without from")
-                    .text()
-                    .expect("Message with non-text from"),
-            ),
-            recipients: vec![doc
-                .get_first(schema.recipients)
-                .expect("Message without recipients")
-                .text()
-                .expect("Message with non-text recipients")
-                .split(",")
-                .map(|s| String::from(s))
-                .collect()],
-            date: doc
-                .get_first(schema.date)
-                .map_or(0, |v: &tantivy::schema::Value| v.u64_value()),
-            original,
             tags,
         }
     }
 }
 
-struct EmailSchema {
+pub struct EmailSchema {
     schema: Schema,
     subject: Field,
     body: Field,
@@ -92,8 +94,8 @@ struct EmailSchema {
     thread: Field,
     id: Field,
     date: Field,
-    original: Field,
     tag: Field,
+    original: Field,
 }
 
 impl Default for EmailSchema {
@@ -103,15 +105,15 @@ impl Default for EmailSchema {
         let body = schema_builder.add_text_field("body", TEXT | STORED);
         let from = schema_builder.add_text_field("from", TEXT | STORED);
         let recipients = schema_builder.add_text_field("recipients", TEXT | STORED);
-        let thread = schema_builder.add_text_field("thread", STRING | STORED);
+        let thread = schema_builder.add_text_field("thread", STRING);
         let id = schema_builder.add_text_field("id", STRING | STORED);
         let tag = schema_builder.add_text_field("tag", STRING | STORED);
-        let original = schema_builder.add_text_field("original", STORED);
         let dateoptions = IntOptions::default()
             .set_fast(Cardinality::SingleValue)
             .set_stored()
             .set_indexed();
         let date = schema_builder.add_u64_field("date", dateoptions);
+        let original = schema_builder.add_text_field("original", TEXT | STORED);
         let schema = schema_builder.build();
         EmailSchema {
             schema,
@@ -122,13 +124,13 @@ impl Default for EmailSchema {
             thread,
             id,
             date,
-            original,
             tag,
+            original,
         }
     }
 }
 impl EmailSchema {
-    pub fn new() -> EmailSchema {
+    pub fn _new() -> EmailSchema {
         EmailSchema::default()
     }
 }
@@ -136,52 +138,22 @@ impl EmailSchema {
 pub struct TantivyStore {
     email: EmailSchema,
     index: tantivy::Index,
-    index_writer: Option<tantivy::IndexWriter>,
+    reader: tantivy::IndexReader,
+    writer: Option<tantivy::IndexWriter>,
     threads: Option<usize>,
     mem_per_thread: Option<usize>,
 }
-impl IMessageStorage for TantivyStore {
-    fn get_message(&self, id: String) -> Result<Message, MessageStoreError> {
-        self.get_message(id.as_str())
-            .ok_or(MessageStoreError::MessageNotFound(
-                "Unable to find message with that id".to_string(),
-            ))
-    }
-    fn add_message(&mut self, msg: Message) -> Result<String, MessageStoreError> {
-        unimplemented!();
-    }
-    fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
-        unimplemented!()
-    }
-    fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
-        unimplemented!()
-    }
-    fn get_messages_page(
-        &self,
-        start: usize,
-        num: usize,
-    ) -> Result<Vec<Message>, MessageStoreError> {
-        Ok(self.latest(num))
-    }
-    fn get_by_date(
-        &self,
-        start: DateTime<Utc>,
-        end: DateTime<Utc>,
-    ) -> Result<Vec<Message>, MessageStoreError> {
-        unimplemented!()
-    }
-}
 impl IMessageSearcher for TantivyStore {
     fn start_indexing_process(&mut self, num: usize) -> Result<(), MessageStoreError> {
-        if self.index_writer.is_none() {
+        if self.writer.is_none() {
-            let index_writer = self.get_index_writer(num)?;
+            let writer = self.get_index_writer(num)?;
-            self.index_writer = Some(index_writer);
+            self.writer = Some(writer);
         }
         Ok(())
     }
 
     fn finish_indexing_process(&mut self) -> Result<(), MessageStoreError> {
-        let writer = &mut self.index_writer;
+        let writer = &mut self.writer;
         match writer {
             Some(writer) => match writer.commit() {
                 Ok(_) => Ok(()),
@@ -202,62 +174,33 @@ impl IMessageSearcher for TantivyStore {
     ) -> Result<String, MessageStoreError> {
         self._add_message(msg, parsed_body)
     }
-    fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError> {
+    fn search_fuzzy(
-        Ok(self.fuzzy(query.as_str(), num))
-    }
-    fn search_by_date(
         &self,
-        start: DateTime<Utc>,
+        query: String,
-        end: DateTime<Utc>,
-    ) -> Result<Vec<Message>, MessageStoreError> {
-        Ok(vec![])
-    }
-    fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
-        Ok(())
-    }
-    fn tag_message_id(
-        &mut self,
-        id: String,
-        tags: HashSet<String>,
-    ) -> Result<usize, MessageStoreError> {
-        let message = self.get_message(id.as_str());
-        match message {
-            Some(mut message) => {
-                let now = Instant::now();
-                self.start_indexing_process(1)?;
-                println!("{}", now.elapsed().as_nanos());
-                self._delete_message(&message)?;
-                println!("{}", now.elapsed().as_nanos());
-                message.tags = tags;
-                let body = message.get_body().clone();
-                self._add_message(message, body.value)?;
-                println!("{}", now.elapsed().as_nanos());
-                self.finish_indexing_process()?;
-                println!("{}", now.elapsed().as_nanos());
-                Ok(1)
-            }
-            None => Err(MessageStoreError::MessageNotFound(
-                "Could not tag message because the message was not found".to_string(),
-            )),
-        }
-    }
-    fn tag_message(
-        &mut self,
-        msg: Message,
-        tags: HashSet<String>,
-    ) -> Result<usize, MessageStoreError> {
-        Ok(1)
-    }
-    fn get_messages_page(
-        &self,
-        start: Message,
         num: usize,
-    ) -> Result<Vec<Message>, MessageStoreError> {
+    ) -> Result<Vec<TantivyMessage>, MessageStoreError> {
-        Ok(vec![])
+        Ok(self.search(query.as_str(), num))
+    }
+    fn delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError> {
+        Ok(())
     }
     fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
         unimplemented!();
     }
+
+    fn latest(&mut self, num: usize) -> Result<Vec<Message>, MessageStoreError> {
+        self._latest(num, None)
+    }
+    fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError> {
+        Ok(self._get_message(id.as_ref()))
+    }
+    fn get_messages_page(
+        &self,
+        start: usize,
+        num: usize,
+    ) -> Result<Vec<Message>, MessageStoreError> {
+        self._latest(num, Some(start))
+    }
 }
 
 impl TantivyStore {
@@ -266,9 +209,14 @@ impl TantivyStore {
     }
     fn _new(path: PathBuf, ro: bool) -> Self {
         let email = EmailSchema::default();
+        let index = TantivyStore::open_or_create_index(path, email.schema.clone());
+        let reader = index
+            .reader()
+            .expect("Unable to create an index reader for this index. Is the index corrupted?");
         TantivyStore {
-            index: TantivyStore::open_or_create_index(path, email.schema.clone()),
+            index,
-            index_writer: None,
+            reader,
+            writer: None,
             email,
             threads: None,
             mem_per_thread: None,
@@ -298,7 +246,7 @@ impl TantivyStore {
         msg: Message,
         parsed_body: String,
     ) -> Result<String, MessageStoreError> {
-        let writer = &mut self.index_writer;
+        let writer = &mut self.writer;
         match writer {
             Some(indexer) => {
                 let mut document = Document::new();
@@ -308,12 +256,13 @@ impl TantivyStore {
                 document.add_text(email.body, parsed_body.as_str());
                 document.add_text(email.from, msg.from.as_str());
                 document.add_text(email.recipients, msg.recipients.join(", ").as_str());
+                document.add_bytes(email.original, msg.original);
                 document.add_u64(email.date, msg.date);
                 msg.tags
-                    .into_iter()
+                    .iter()
                     .for_each(|t| document.add_text(email.tag, t.as_str()));
                 indexer.add_document(document);
-                Ok(msg.id)
+                Ok(msg.id.clone())
             }
             None => Err(MessageStoreError::CouldNotAddMessage(
                 "No indexer was allocated".to_string(),
@@ -321,7 +270,7 @@ impl TantivyStore {
         }
     }
     fn _delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError> {
-        let writer = &mut self.index_writer;
+        let writer = &mut self.writer;
         match writer {
             Some(indexer) => {
                 let term = Term::from_field_text(self.email.id, msg.id.as_ref());
@@ -333,13 +282,13 @@ impl TantivyStore {
             )),
         }
     }
-    pub fn tag_doc(&self, doc: Document, tags: Vec<String>) -> Result<(), tantivy::TantivyError> {
+    pub fn _tag_doc(&self, doc: Document, tags: Vec<String>) -> Result<(), tantivy::TantivyError> {
-        let mut index_writer = self.get_index_writer(1).ok().unwrap();
+        let mut writer = self.get_index_writer(1).ok().unwrap();
-        let id = TantivyMessage::from(doc, &self.email).id;
+        let id = TantivyMessage::from_tantivy(doc, &self.email).id;
         let term = Term::from_field_text(self.email.id, id.as_ref());
-        index_writer.delete_term(term.clone());
+        writer.delete_term(term.clone());
-        index_writer.commit()?;
+        writer.commit()?;
-        self.index.load_searchers()
+        self.reader.reload()
     }
 
     fn get_index_writer(
@@ -377,7 +326,7 @@ impl TantivyStore {
             }
         };
         info!(
-            "For your information, we're using {} threads with {}mb memory per thread",
+            "We're using {} threads with {}mb memory per thread",
             num_cpu,
             mem_per_thread / BYTES_IN_MB
         );
@@ -392,30 +341,33 @@ impl TantivyStore {
         }
     }
 
-    pub fn latest(&self, num: usize) -> Vec<TantivyMessage> {
+    pub fn _latest(
-        let searcher = self.index.searcher();
+        &self,
-        let docs = searcher
+        num: usize,
+        _skip: Option<usize>,
+    ) -> Result<Vec<TantivyMessage>, MessageStoreError> {
+        let searcher = self.reader.searcher();
+        let skip = _skip.unwrap_or(0);
+        let mut docs = searcher
             .search(
                 &AllQuery,
-                &TopDocs::with_limit(num).order_by_field::<u64>(self.email.date),
+                &TopDocs::with_limit(num + skip).order_by_u64_field(self.email.date),
             )
-            .unwrap();
+            .map_err(|e| MessageStoreError::CouldNotGetMessages(vec![]))?;
         let mut ret = vec![];
-        for doc in docs {
+        let page = docs
+            .drain(skip..)
+            .collect::<Vec<(u64, tantivy::DocAddress)>>();
+        for doc in page {
             let retrieved_doc = searcher.doc(doc.1).unwrap();
-            ret.push(TantivyMessage::from(retrieved_doc, &self.email));
+            ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
         }
-        ret
+        Ok(ret)
     }
 
-    pub fn by_date(&self) {
+    pub fn get_doc(&self, id: &str) -> Result<Document, tantivy::TantivyError> {
-        let searcher = self.index.searcher();
+        // Is this needed? self.reader.load_searchers()?;
-        let docs = RangeQuery::new_u64(self.email.date, 1522704682..1524704682);
+        let searcher = self.reader.searcher();
-        let numdocs = searcher.search(&docs, &Count).unwrap();
-    }
-    pub fn get_doc(&self, id: &str) -> Result<Document, tantivy::Error> {
-        self.index.load_searchers()?;
-        let searcher = self.index.searcher();
         let termq = TermQuery::new(
             Term::from_field_text(self.email.id, id.as_ref()),
             IndexRecordOption::Basic,
@@ -424,7 +376,7 @@ impl TantivyStore {
         match addr {
             Ok(doc) => match doc.first() {
                 Some((_score, doc_address)) => searcher.doc(*doc_address),
-                None => Err(tantivy::Error::InvalidArgument(
+                None => Err(tantivy::TantivyError::InvalidArgument(
                     "Document not found".to_string(),
                 )),
             },
@@ -432,27 +384,24 @@ impl TantivyStore {
             Err(e) => Err(e),
         }
     }
-    pub fn get_message(&self, id: &str) -> Option<TantivyMessage> {
+    pub fn _get_message(&self, id: &str) -> Option<TantivyMessage> {
         let doc = self.get_doc(id);
         match doc {
-            Ok(doc) => Some(TantivyMessage::from(doc, &self.email)),
+            Ok(doc) => Some(TantivyMessage::from_tantivy(doc, &self.email)),
             Err(_) => None,
         }
     }
     pub fn search(&self, text: &str, num: usize) -> Vec<TantivyMessage> {
-        let searcher = self.index.searcher();
+        let searcher = self.reader.searcher();
         let term = Term::from_field_text(self.email.subject, text);
         let term_body = Term::from_field_text(self.email.body, text);
-        let query = TermQuery::new(term, IndexRecordOption::Basic);
+        let top_docs_by_date = TopDocs::with_limit(num).order_by_u64_field(self.email.date);
-        let query_body = TermQuery::new(term_body, IndexRecordOption::Basic);
+        let bquery = BooleanQuery::new_multiterms_query(vec![term, term_body]);
-        let top_docs_by_date = TopDocs::with_limit(num).order_by_field::<u64>(self.email.date);
-        let queries: Vec<(Occur, Box<Query>)> = vec![];
-        let bquery = BooleanQuery::from(queries);
         let top_docs = searcher.search(&bquery, &top_docs_by_date).unwrap();
         let mut ret = vec![];
         for doc in top_docs {
             let retrieved_doc = searcher.doc(doc.1).unwrap();
-            ret.push(TantivyMessage::from(retrieved_doc, &self.email));
+            ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
         }
         ret
     }
@@ -460,7 +409,7 @@ impl TantivyStore {
     pub fn fuzzy(&self, text: &str, num: usize) -> Vec<TantivyMessage> {
         let mut terms = text.split(' ').collect::<Vec<&str>>();
         terms.insert(0, text);
-        let searcher = self.index.searcher();
+        let searcher = self.reader.searcher();
         let mut ret = self.search(text, num);
         for n in 1..2 {
             if ret.len() < num {
@@ -473,13 +422,12 @@ impl TantivyStore {
                     queries.push((Occur::Should, Box::new(query)));
                     queries.push((Occur::Should, Box::new(query_body)));
                 }
-                let top_docs_by_date =
+                let top_docs_by_date = TopDocs::with_limit(num).order_by_u64_field(self.email.date);
-                    TopDocs::with_limit(num).order_by_field::<u64>(self.email.date);
                 let bquery = BooleanQuery::from(queries);
                 let top_docs = searcher.search(&bquery, &top_docs_by_date).unwrap();
                 for doc in top_docs {
                     let retrieved_doc = searcher.doc(doc.1).unwrap();
-                    ret.push(TantivyMessage::from(retrieved_doc, &self.email));
+                    ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
                 }
             }
         }

src/stores/message_store.rs

@@ -1,7 +1,7 @@
-use crate::message::{get_id, Body, Message, Mime};
+use crate::message::{Message, ShortMessage};
 use crate::stores::{IMessageSearcher, IMessageStorage, IMessageStore, MessageStoreError};
 use chrono::{DateTime, Utc};
-use log::{debug, error, info};
+use log::error;
 use maildir::{MailEntry, Maildir};
 use pbr::{MultiBar, ProgressBar};
 use rayon::prelude::*;
@@ -9,25 +9,17 @@ use std::collections::HashSet;
 use std::path::PathBuf;
 use std::sync::mpsc;
 use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
-pub struct MessageStore<I, S>
+pub struct MessageStore {
-where
+    pub searcher: Box<dyn IMessageSearcher>,
-    I: IMessageSearcher,
+    pub storage: Option<Box<dyn IMessageStorage>>,
-    S: IMessageStorage,
-{
-    pub searcher: Box<I>,
-    pub storage: Box<S>,
     progress: Option<ProgressBar<pbr::Pipe>>,
     display_progress: bool,
 }
-impl<I, S> IMessageStore for MessageStore<I, S>
+impl IMessageStore for MessageStore {
-where
+    fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError> {
-    I: IMessageSearcher,
+        self.searcher.get_message(id)
-    S: IMessageStorage,
-{
-    fn get_message(&self, id: String) -> Result<Message, MessageStoreError> {
-        self.storage.get_message(id)
     }
 
     fn add_message(
@@ -35,7 +27,9 @@ where
         msg: Message,
         parsed_body: String,
     ) -> Result<String, MessageStoreError> {
-        self.searcher.add_message(msg, parsed_body)
+        let id = msg.id.clone();
+        self.searcher.add_message(msg, parsed_body)?;
+        Ok(id)
     }
 
     fn add_maildir(&mut self, path: PathBuf, all: bool) -> Result<usize, MessageStoreError> {
@@ -46,7 +40,7 @@ where
         id: String,
         tags: HashSet<String>,
     ) -> Result<usize, MessageStoreError> {
-        self.searcher.tag_message_id(id, tags)
+        unimplemented!();
     }
 
     fn tag_message(
@@ -65,7 +59,7 @@ where
         start: usize,
         num: usize,
     ) -> Result<Vec<Message>, MessageStoreError> {
-        self.storage.get_messages_page(start, num)
+        self.searcher.get_messages_page(start, num)
     }
 
     fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError> {
@@ -82,12 +76,12 @@ where
         unimplemented!();
     }
 }
-impl<I, S> MessageStore<I, S>
+impl MessageStore {
-where
+    pub fn new(
-    I: IMessageSearcher,
+        searcher: Box<dyn IMessageSearcher>,
-    S: IMessageStorage,
+        storage: Option<Box<dyn IMessageStorage>>,
-{
+        display_progress: bool,
-    pub fn new(searcher: Box<I>, storage: Box<S>, display_progress: bool) -> Self {
+    ) -> Self {
         MessageStore {
             searcher,
             storage,
@@ -119,7 +113,7 @@ where
         let mut mb = MultiBar::new();
         mb.println(&format!("Indexing {} emails", num));
         let mut index_bar = mb.create_bar(num as u64);
-        if num < 10000000 {
+        if num < 10_000_000 {
             mb.println("This will take no time!");
         }
         index_bar.message("Indexed ");
@@ -155,8 +149,8 @@ where
             mails
                 .into_par_iter()
                 .for_each_with(tx, |tx, msg| match msg {
-                    Ok(mut unparsed_msg) => {
+                    Ok(unparsed_msg) => {
-                        let message = Message::from_mailentry(&mut unparsed_msg);
+                        let message = Message::from_mailentry(unparsed_msg);
                         match message {
                             Ok(msg) => {
                                 let parsed_body = msg.get_body().as_text();

src/stores/mod.rs

@@ -6,15 +6,16 @@ mod _impl;
 use _impl::rocksdb;
 use _impl::tantivy;
 use std::fmt;
-mod message_store;
+pub mod message_store;
 use message_store::MessageStore;
+use std::time::Instant;
 
 pub enum Searchers {
     Tantivy(PathBuf),
 }
 pub enum Storages {
-    Tantivy(PathBuf),
     Rocksdb(PathBuf),
+    Tantivy(PathBuf),
 }
 
 pub struct MessageStoreBuilder {
@@ -51,7 +52,7 @@ impl MessageStoreBuilder {
         }
     }
 
-    pub fn new_from_cfg(pathbuf: PathBuf) -> MessageStoreBuilder {
+    pub fn new_from_cfg(_pathbuf: PathBuf) -> MessageStoreBuilder {
         unimplemented!();
     }
 
@@ -74,24 +75,31 @@ impl MessageStoreBuilder {
         self.storage = Some(storage);
         self
     }
+    pub fn maildir(&mut self, path: PathBuf) -> &mut Self {
+        self.maildir_path = Some(path);
+        self
+    }
 
-    pub fn build(&self) -> Result<Box<IMessageStore>, MessageStoreBuilderError> {
+    pub fn build(&self) -> Result<impl IMessageStore, MessageStoreBuilderError> {
-        let store = match &self.storage {
+        //let storage: Result<Option<Box<dyn IMessageStorage>>, MessageStoreBuilderError> =
-            None => Err(MessageStoreBuilderError::CouldNotCreateStoreError(
+        //    match &self.storage {
-                "No store type was provided".to_string(),
+        //        None => Ok(None),
-            )),
+        //        Some(store_type) => match store_type {
-            Some(store_type) => match store_type {
+        //            Storages::Rocksdb(path) => {
-                Storages::Tantivy(path) => match self.read_only {
+        //                let mut p = path.clone();
-                    true => Ok(tantivy::TantivyStore::new_ro(std::path::PathBuf::from(
+        //                p.push("storage");
-                        path,
+        //                Ok(Some(Box::new(rocksdb::RocksDBStore::new(p))))
-                    ))),
+        //            }
-                    false => Ok(tantivy::TantivyStore::new(std::path::PathBuf::from(path))),
+        //            Storages::Tantivy(path) => {
-                },
+        //                let mut p = path.clone();
-                Storages::Rocksdb => Err(MessageStoreBuilderError::CouldNotCreateStoreError(
+        //                p.push("storage");
-                    "Rocksdb is not yet supported, try again later".to_string(),
+        //                tantivy = Some(tantivy::TantivyStore::new(
-                )),
+        //                    std::path::PathBuf::from(p),
-            },
+        //                ));
-        }?;
+        //                Ok(Box::new(tantivy))
+        //            }
+        //        };
+        //        },
 
         let searcher = match &self.searcher {
             None => Err(MessageStoreBuilderError::CouldNotCreateStoreError(
@@ -99,17 +107,14 @@ impl MessageStoreBuilder {
             )),
             Some(searcher_type) => match searcher_type {
                 Searchers::Tantivy(path) => {
-                    Ok(tantivy::TantivyStore::new(std::path::PathBuf::from(path)))
+                    let mut p = path.clone();
+                    p.push("searcher");
+                    Ok(tantivy::TantivyStore::new(std::path::PathBuf::from(p)))
                 }
             },
-        }?;
+        };
 
-        Ok(Box::new(MessageStore::<
+        Ok(MessageStore::new(Box::new(searcher?), None, !self.debug))
-            tantivy::TantivyStore,
-            tantivy::TantivyStore,
-        >::new(
-            Box::new(searcher), Box::new(store), !self.debug
-        )))
     }
 }
 
@@ -118,7 +123,9 @@ pub enum MessageStoreError {
     CouldNotAddMessage(String),
     CouldNotOpenMaildir(String),
     CouldNotModifyMessage(String),
+    CouldNotGetMessage(String),
     CouldNotGetMessages(Vec<String>),
+    CouldNotConvertMessage(String),
     InvalidQuery(String),
 }
 
@@ -134,7 +141,11 @@ impl fmt::Display for MessageStoreError {
             MessageStoreError::CouldNotGetMessages(s) => {
                 format!("Could not get messages {}", s.join(", "))
             }
+            MessageStoreError::CouldNotGetMessage(s) => format!("Could not get message {}", s),
             MessageStoreError::InvalidQuery(s) => format!("Could query message {}", s),
+            MessageStoreError::CouldNotConvertMessage(s) => {
+                format!("Could not convert message {}", s)
+            }
         };
         write!(f, "Message Store Error {}", msg)
     }
@@ -147,37 +158,24 @@ pub trait IMessageSearcher {
         parsed_body: String,
     ) -> Result<String, MessageStoreError>;
     fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError>;
-    fn search_by_date(
-        &self,
-        start: DateTime<Utc>,
-        end: DateTime<Utc>,
-    ) -> Result<Vec<Message>, MessageStoreError>;
 
-    fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError>;
+    fn delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError>;
-    fn tag_message_id(
-        &mut self,
-        id: String,
-        tags: HashSet<String>,
-    ) -> Result<usize, MessageStoreError>;
-    fn tag_message(
-        &mut self,
-        msg: Message,
-        tags: HashSet<String>,
-    ) -> Result<usize, MessageStoreError>;
 
-    fn get_messages_page(
-        &self,
-        start: Message,
-        num: usize,
-    ) -> Result<Vec<Message>, MessageStoreError>;
     fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError>;
     fn start_indexing_process(&mut self, num: usize) -> Result<(), MessageStoreError>;
     fn finish_indexing_process(&mut self) -> Result<(), MessageStoreError>;
+    fn latest(&mut self, num: usize) -> Result<Vec<Message>, MessageStoreError>;
+    fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError>;
+    fn get_messages_page(
+        &self,
+        start: usize,
+        num: usize,
+    ) -> Result<Vec<Message>, MessageStoreError>;
 }
 
 pub trait IMessageStorage {
-    fn get_message(&self, id: String) -> Result<Message, MessageStoreError>;
+    fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError>;
-    fn add_message(&mut self, msg: Message) -> Result<String, MessageStoreError>;
+    fn add_message(&mut self, msg: &Message) -> Result<String, MessageStoreError>;
     fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError>;
     fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError>;
     fn get_messages_page(
@@ -193,7 +191,7 @@ pub trait IMessageStorage {
 }
 
 pub trait IMessageStore {
-    fn get_message(&self, id: String) -> Result<Message, MessageStoreError>;
+    fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError>;
     fn add_message(
         &mut self,
         msg: Message,

src/terminal/events.rs

@@ -15,8 +15,8 @@ pub enum Event<I> {
 /// type is handled in its own thread and returned to a common `Receiver`
 pub struct Events {
     rx: mpsc::Receiver<Event<Key>>,
-    input_handle: thread::JoinHandle<()>,
+    pub input_handle: thread::JoinHandle<()>,
-    tick_handle: thread::JoinHandle<()>,
+    pub tick_handle: thread::JoinHandle<()>,
 }
 
 #[derive(Debug, Clone, Copy)]

src/terminal/input/mod.rs

@@ -6,15 +6,15 @@ mod reader;
 mod search;
 
 pub struct InputHandler {
-    name: String,
+    pub name: String,
     pre: bool,
     f: Box<Runnable>,
     children: Vec<Box<InputHandler>>,
 }
 
 #[derive(Debug)]
-pub struct NoopRunner {}
+pub struct _NoopRunner {}
-impl Runnable for NoopRunner {
+impl Runnable for _NoopRunner {
     fn run(&self, _e: &Event<Key>, _store: &mut Store) -> bool {
         false
     }

src/terminal/input/reader.rs

@@ -12,39 +12,35 @@ impl Runnable for ReaderRunner {
                 Event::Input(key) => match key {
                     Key::Esc | Key::Char('q') => {
                         store.reader_store.read(None);
-                        return true;
+                        true
                     }
                     Key::Char('j') | Key::Down => {
                         store.reader_store.scroll(3);
-                        return true;
+                        true
                     }
                     Key::Char('k') | Key::Up => {
                         store.reader_store.scroll(-3);
-                        return true;
+                        true
                     }
                     Key::Ctrl('u') | Key::PageUp => {
                         store.reader_store.scroll(-20);
-                        return true;
+                        true
                     }
                     Key::Ctrl('d') | Key::PageDown => {
                         store.reader_store.scroll(20);
-                        return true;
+                        true
                     }
                     Key::Char('t') => {
                         store.tags_store.edit(store.reader_store.get_message());
-                        return true;
+                        true
                     }
                     Key::Home => {
                         store.reader_store.scroll_top();
-                        return true;
+                        true
-                    }
-                    _ => {
-                        return false;
                     }
+                    _ => false,
                 },
-                _ => {
+                _ => false,
-                    return false;
-                }
             }
         } else {
             false

src/terminal/mod.rs

@@ -5,10 +5,8 @@ mod views;
 use crate::stores::{MessageStoreBuilder, Searchers, Storages};
 use events::Events;
 use input::{handlers, run};
-use std::cell::RefCell;
 use std::io;
 use std::path::PathBuf;
-use std::rc::Rc;
 use store::Store;
 use termion::raw::IntoRawMode;
 use tui::backend::TermionBackend;
@@ -38,10 +36,10 @@ pub fn start(index: PathBuf) -> Result<(), io::Error> {
                     break;
                 };
             }
-            terminal.clear();
+            terminal.clear()?;
         }
         Err(e) => {
-            terminal.clear();
+            terminal.clear()?;
             println!("Error {}", e);
         }
     };

src/terminal/store/list.rs

@@ -7,11 +7,11 @@ pub struct ListStore<'a> {
     pub page_size: usize,
     pub curr_idx: usize,
     pub fetched_first: bool,
-    pub message_store: &'a Box<IMessageStore>,
+    pub message_store: &'a IMessageStore,
 }
 
 impl<'a> ListStore<'a> {
-    pub fn new(msg_store: &'a Box<IMessageStore>) -> ListStore<'a> {
+    pub fn new(msg_store: &'a IMessageStore) -> ListStore<'a> {
         ListStore {
             messages: vec![],
             selected: 0,
@@ -22,12 +22,6 @@ impl<'a> ListStore<'a> {
         }
     }
 
-    pub fn set_results(&mut self, messages: Vec<Message>) -> &Self {
-        self.messages = messages;
-        self.set_selected(0);
-        self
-    }
-
     pub fn get_selected(&mut self) -> Option<&Message> {
         self.messages.get(self.selected)
     }

src/terminal/store/mod.rs

@@ -12,16 +12,16 @@ pub struct Store<'a> {
     pub exit: bool,
     pub list_store: ListStore<'a>,
     pub search_store: SearchStore<'a>,
-    pub reader_store: ReaderStore,
+    pub reader_store: ReaderStore<'a>,
     pub tags_store: TagsStore<'a>,
 }
 impl<'a> Store<'a> {
-    pub fn new(message_store: &'a Box<IMessageStore>) -> Store {
+    pub fn new(message_store: &'a IMessageStore) -> Store {
         Store {
             exit: false,
             search_store: SearchStore::new(message_store),
             list_store: ListStore::new(message_store),
-            reader_store: ReaderStore::new(),
+            reader_store: ReaderStore::new(message_store),
             tags_store: TagsStore::new(message_store),
         }
     }

src/terminal/store/reader.rs

@@ -1,16 +1,19 @@
 use crate::message::Message;
+use crate::stores::IMessageStore;
 use std::cmp::max;
 
-pub struct ReaderStore {
+pub struct ReaderStore<'a> {
     pub message: Option<Message>,
     pub scroll: u16,
+    pub storage: &'a IMessageStore,
 }
 
-impl ReaderStore {
+impl<'a> ReaderStore<'a> {
-    pub fn new() -> ReaderStore {
+    pub fn new(storage: &'a IMessageStore) -> ReaderStore<'a> {
         ReaderStore {
             message: None,
             scroll: 0,
+            storage,
         }
     }
 
@@ -21,8 +24,14 @@ impl ReaderStore {
         }
     }
     pub fn read(&mut self, msg: Option<&Message>) {
-        self.message = msg.cloned();
+        let msg = msg.cloned();
-        self.scroll = 0;
+        match msg {
+            Some(msg) => {
+                self.message = self.storage.get_message(msg.id).ok().unwrap();
+                self.scroll = 0;
+            }
+            None => self.message = None,
+        }
     }
 
     pub fn scroll_top(&mut self) {

src/terminal/store/search.rs

@@ -4,11 +4,11 @@ use crate::stores::IMessageStore;
 pub struct SearchStore<'a> {
     pub search_term: String,
     pub searching: bool,
-    pub searcher: &'a Box<IMessageStore>,
+    pub searcher: &'a IMessageStore,
     pub results: Vec<Message>,
 }
 impl<'a> SearchStore<'a> {
-    pub fn new(msg_store: &'a Box<IMessageStore>) -> SearchStore {
+    pub fn new(msg_store: &'a IMessageStore) -> SearchStore {
         SearchStore {
             search_term: String::from(""),
             searching: false,

src/terminal/store/tags.rs

@@ -2,11 +2,11 @@ use crate::message::Message;
 use crate::stores::IMessageStore;
 
 pub struct TagsStore<'a> {
-    pub message_store: &'a Box<IMessageStore>,
+    pub message_store: &'a IMessageStore,
     pub message: Option<Message>,
 }
 impl<'a> TagsStore<'a> {
-    pub fn new(msg_store: &'a Box<IMessageStore>) -> TagsStore<'a> {
+    pub fn new(msg_store: &'a IMessageStore) -> TagsStore<'a> {
         TagsStore {
             message: None,
             message_store: msg_store,

src/terminal/views/email_read.rs

@@ -1,7 +1,6 @@
 use crate::message::Message;
 use tui::backend::Backend;
-use tui::layout::{Alignment, Rect};
+use tui::layout::Rect;
-use tui::layout::{Constraint, Direction, Layout};
 use tui::style::{Modifier, Style};
 use tui::widgets::{Block, Borders, Paragraph, Text, Widget};
 use tui::Frame;
@@ -19,7 +18,7 @@ pub fn draw<B: Backend>(f: &mut Frame<B>, message: &Message, scroll: u16) {
 
     let block = Block::default()
         .borders(Borders::ALL)
-        .title_style(Style::default().modifier(Modifier::Bold));
+        .title_style(Style::default().modifier(Modifier::BOLD));
     Paragraph::new(text.iter())
         .block(block.clone())
         .wrap(true)

src/terminal/views/search_results.rs

@@ -1,9 +1,8 @@
 use crate::terminal::store::Store;
-use std::io;
+use tui::backend::Backend;
-use tui::backend::{Backend, TermionBackend};
+use tui::layout::Rect;
-use tui::layout::{Constraint, Direction, Layout, Rect};
 use tui::style::{Color, Modifier, Style};
-use tui::widgets::{Block, Borders, List, SelectableList, Text, Widget};
+use tui::widgets::{Block, Borders, SelectableList, Widget};
 use tui::Frame;
 
 pub fn draw<B: Backend>(f: &mut Frame<B>, area: Rect, store: &Store) {
@@ -23,7 +22,7 @@ pub fn draw<B: Backend>(f: &mut Frame<B>, area: Rect, store: &Store) {
         )
         .select(Some(store.list_store.selected))
         .style(style)
-        .highlight_style(style.fg(Color::LightGreen).modifier(Modifier::Bold))
+        .highlight_style(style.fg(Color::LightGreen).modifier(Modifier::BOLD))
         .highlight_symbol(">")
         .render(f, area);
 }