rms/rms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Saving non-working state with temporary readme

Browse Source
This commit is contained in:
Lewis Diamond 2020-07-31 21:56:00 -04:00
parent e0111f723c
commit f74963cf3c
27 changed files with 1994 additions and 11418 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -1,3 +1,5 @@
/target
/idx
/idx*
/.err
**/*.rs.bk
/html_emails

2400
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

46
Cargo.toml
View File

@ -7,24 +7,30 @@ edition = "2018"
[dependencies]
#maildir = { git = "https://github.com/lewisdiamond/maildir.git" }
maildir = { path = "/home/ldiamond/dev/maildir/" }
#html2text = "0.1.8"
html2text = { git = "https://github.com/lewisdiamond/rust-html2text.git"}
mailparse = "0.6.5"
rayon = "1.0.3"
tantivy = "0.8.1"
html2text = "0.1.13"
#html2text = { git = "https://github.com/lewisdiamond/rust-html2text.git"}
mailparse = "0.13.0"
rayon = "1.3.1"
tantivy = "0.12.0"
tempdir = "0.3.7"
serde_json = "1.0.33"
structopt = "0.2.14"
shellexpand = "1.0"
log = { version = "0.4", features = ["max_level_debug", "release_max_level_warn"] }
pretty_env_logger = "0.3"
pbr = "1.0.1"
num_cpus = "1.9.0"
sys-info = "0.5.6"
tui = "0.4.0"
termion = "1.5.1"
chrono = "0.4.6"
sha2 = "0.8.0"
html5ever = "*"
rocksdb = "0.12.0"
serde = { version = "1.0.89", features = ["derive"]}
serde_json = "1.0.57"
serde = { version = "1.0.114", features = ["derive"] }
structopt = "0.3.15"
shellexpand = "2.0.0"
log = { version = "0.4.11", features = ["max_level_debug", "release_max_level_warn"] }
pretty_env_logger = "0.4.0"
pbr = "1.0.3"
num_cpus = "1.13.0"
sys-info = "0.7.0"
tui = "0.10.0"
termion = "1.5.5"
chrono = "0.4.13"
sha2 = "0.9.1"
html5ever = "0.25.1"
rocksdb = { path = "../rust-rocksdb/" }
jemallocator = "0.3.2"
#maildir = "0.4.2"
select = "0.5.0"
[dev-dependencies]
rand = "0.7.3"

8
README.md Normal file
View File

@ -0,0 +1,8 @@
# This project is not currently in a working state. Contributions are welcome.
RMS is an email system that allows you to easily search, tag, list and read your
emails from the command line.
It is similar to NotMuch but much faster and provides more ways to access your
emails.

9784
output
View File

File diff suppressed because it is too large Load Diff

22
src/bin/readmail.rs
View File

@ -1,28 +1,24 @@
use log::{debug, error};
use rms::message::{get_id, Message};
use rms::readmail::{extract_body, html2text};
use log::error;
use rms::message::Message;
use rms::readmail::html2text;
mod readmail_cmd;
use mailparse::*;
use readmail_cmd::source;
use rms::message::Mime;
use std::io::BufRead;
fn main() {
let src: Box<BufRead> = source();
let src: Box<dyn BufRead> = source();
let b_msg_rslt = src.split(3);
for m in b_msg_rslt {
match m {
Ok(buf) => {
let hash = get_id(&buf);
if let Ok(mut msg) = parse_mail(buf.as_slice()) {
let message = Message::from_parsedmail(&mut msg, hash);
let message = Message::from_data(buf);
match message {
Ok(message) => {
println!("From: {}", message.from);
println!("To: {}", message.recipients.join(", "));
println!("Subject: {}", message.subject);
let body = extract_body(&mut msg, false);
for b in body {
for b in message.body {
println!("Body Mime: {}", b.mime.as_str());
match b.mime {
Mime::PlainText => println!("\n\n{}", b.value),
@ -31,11 +27,7 @@ fn main() {
}
}
}
Err(e) => error!("Failed to make sense of the message"),
}
} else {
error!("Failed to parse the file");
::std::process::exit(1);
Err(_e) => error!("Failed to make sense of the message"),
}
}
Err(e) => {

3
src/bin/readmail_cmd/mod.rs
View File

@ -18,6 +18,9 @@ pub fn source() -> Box<BufRead> {
.args(&[Arg::from_usage(
"[input] 'Read from a file, or stdin if omitted'",
)])
.args(&[Arg::from_usage(
"[-o destination] 'Save attachment to destination'",
)])
.get_matches();
match matches.value_of("input") {

70
src/bin/rms.rs
View File

@ -1,8 +1,10 @@
use log::{error, info, trace};
use rms::cmd::{opts, Command, OutputType};
use rms::stores::{MessageStoreBuilder, Searchers, Storages};
use rms::stores::{IMessageStore, MessageStoreBuilder, Searchers, Storages};
use rms::terminal;
use std::collections::HashSet;
use std::io::{self, Write};
use std::time::Instant;
fn main() {
pretty_env_logger::init();
@ -54,6 +56,7 @@ fn main() {
//message_store.index_mails(full);
}
Command::Search { term, output, num } => {
let now = Instant::now();
let message_store = MessageStoreBuilder::new()
.storage(Storages::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path.clone()))
@ -72,6 +75,12 @@ fn main() {
OutputType::Full => {
println!("{:?}", results);
}
OutputType::Raw => {
let mut out = io::stdout();
for result in results {
out.write_all(result.original.as_ref()).unwrap();
}
}
}
}
Err(e) => error!("{}", e),
@ -99,29 +108,64 @@ fn main() {
match message_store {
Ok(store) => {
let result = store.get_message(id).ok().unwrap();
match output {
let result = store.get_message(id);
match result {
Ok(Some(good_msg)) => match output {
OutputType::Short => {
println!("{:?} | {}", result.id, result.subject);
println!("{} | {}", good_msg.id, good_msg.subject);
}
OutputType::Raw => {
io::stdout().write_all(good_msg.original.as_ref()).unwrap();
}
OutputType::Full => {
println!("{:?}", result);
println!("From: {}", good_msg.from);
println!(
"To: {}",
good_msg
.recipients
.get(0)
.unwrap_or(&String::from("Unknown"))
);
println!("Subject: {}", good_msg.subject);
println!(
"{}",
good_msg
.body
.first()
.map(|b| b.value.clone())
.unwrap_or(String::from("No body"))
);
}
},
Ok(None) => error!("Message not found"),
Err(e) => error!("ERROR {}", e),
}
}
}
Err(e) => error!("{}", e),
Err(e) => error!("Store isn't right... {}", e),
}
}
Command::Interactive {} => {
terminal::start(index_dir_path).unwrap();
}
Command::Latest { num: _num } => {
let _message_store = MessageStoreBuilder::new().build(); //maildir_path[0].clone(), index_dir_path);
//let searcher = Searcher::new(index_dir_path);
//let stuff = searcher.latest(num, None);
//for s in stuff {
// println!("{}", s.date);
//}
let message_store = MessageStoreBuilder::new()
.storage(Storages::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path.clone()))
.build();
match message_store {
Ok(store) => {
let page = store.get_messages_page(0, _num);
match page {
Ok(msgs) => {
for m in msgs {
println!("{}", m.id);
}
}
Err(e) => println!("Could not read messages, {}", e),
}
}
Err(e) => println!("Could not load the index, {}", e),
}
}
Command::Tag { id, tags } => {
let message_store = MessageStoreBuilder::new()

13
src/cmd/mod.rs
View File

@ -18,6 +18,7 @@ pub fn expand_path(input: &OsStr) -> PathBuf {
pub enum OutputType {
Short,
Full,
Raw,
}
#[derive(Debug)]
@ -49,6 +50,7 @@ impl FromStr for OutputType {
match input.to_lowercase().as_str() {
"short" => Ok(OutputType::Short),
"full" => Ok(OutputType::Full),
"raw" => Ok(OutputType::Raw),
_ => Err(OutputTypeError::UnknownTypeError),
}
}
@ -64,7 +66,7 @@ impl FromStr for OutputType {
)]
pub struct Opt {
#[structopt(
parse(from_os_str = "expand_path"),
parse(from_os_str = expand_path),
short,
long,
env = "RMS_CONFIG_PATH",
@ -73,7 +75,7 @@ pub struct Opt {
pub config: PathBuf,
#[structopt(
parse(from_os_str = "expand_path"),
parse(from_os_str = expand_path),
short,
long,
env = "RMS_INDEX_DIR_PATH"
@ -89,7 +91,7 @@ pub enum Command {
#[structopt(name = "index", rename_all = "kebab-case")]
Index {
#[structopt(
parse(from_os_str = "expand_path"),
parse(from_os_str = expand_path),
short,
long,
required = true,
@ -117,11 +119,10 @@ pub enum Command {
#[structopt(rename_all = "kebab-case")]
Get {
#[structopt(short, long)]
id: String,
#[structopt(short, long, default_value = "short")]
output: OutputType,
id: String,
},
#[structopt(rename_all = "kebab-case")]

8
src/lib.rs
View File

@ -6,9 +6,7 @@ extern crate num_cpus;
extern crate pbr;
extern crate pretty_env_logger;
extern crate rayon;
#[macro_use]
extern crate serde;
#[macro_use]
extern crate serde_json;
extern crate shellexpand;
extern crate structopt;
@ -22,3 +20,9 @@ pub mod message;
pub mod readmail;
pub mod stores;
pub mod terminal;
extern crate jemallocator;
#[cfg(test)]
extern crate rand;
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

107
src/message/mod.rs
View File

@ -1,16 +1,16 @@
use crate::readmail;
use crate::readmail::html2text;
use chrono::prelude::*;
use html2text::from_read;
use log::{debug, error};
use maildir::MailEntry;
use mailparse::{dateparse, ParsedMail};
use maildir::{MailEntry, ParsedMailEntry};
use mailparse::{dateparse, parse_mail, ParsedMail};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha512};
use std::collections::HashSet;
use std::convert::AsRef;
use std::path::{Path, PathBuf};
use std::string::ToString;
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum Mime {
PlainText,
Html,
@ -39,7 +39,7 @@ impl Mime {
}
}
}
#[derive(Debug, Default, Clone)]
#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
pub struct Body {
pub mime: Mime,
pub value: String,
@ -57,7 +57,7 @@ impl Body {
}
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Message {
pub id: String,
pub body: Vec<Body>,
@ -65,10 +65,18 @@ pub struct Message {
pub from: String,
pub recipients: Vec<String>,
pub date: u64,
pub original: Option<String>,
pub original: Vec<u8>,
pub tags: HashSet<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ShortMessage {
pub id: String,
pub subject: String,
pub from: String,
pub date: u64,
}
pub struct MessageBuilder {
body: Option<String>,
subject: Option<String>,
@ -76,12 +84,19 @@ pub struct MessageBuilder {
recipients: Option<Vec<String>>,
date: Option<u64>,
id: Option<String>,
original: Option<Vec<u8>>,
}
pub fn get_id(data: &Vec<u8>) -> String {
let mut hasher = Sha512::default();
hasher.input(data);
format!("{:x}", hasher.result())
format!("{:x}", Sha512::digest(data))
}
impl ToString for ShortMessage {
fn to_string(&self) -> String {
let dt = Local.timestamp(self.date as i64, 0);
let dstr = dt.format("%a %b %e %T %Y").to_string();
format!("{}: [{}] {}", dstr, self.from, self.subject.as_str())
}
}
impl ToString for Message {
@ -119,8 +134,20 @@ pub struct MessageError {
pub message: String,
}
impl MessageError {
pub fn from(msg: &str) -> Self {
MessageError {
message: String::from(msg),
}
}
}
impl Message {
pub fn from_parsedmail(mut msg: &mut ParsedMail, id: String) -> Result<Self, MessageError> {
pub fn from_parsedmail(
msg: &ParsedMail,
id: String,
original: Vec<u8>,
) -> Result<Self, MessageError> {
let headers = &msg.headers;
let mut subject: String = "".to_string();
let mut from: String = "".to_string();
@ -128,18 +155,16 @@ impl Message {
let default_date = 0;
let mut date = default_date;
for h in headers {
if let Ok(s) = h.get_key() {
match s.as_ref() {
"Subject" => subject = h.get_value().unwrap_or("".to_string()),
"From" => from = h.get_value().unwrap_or("".to_string()),
"To" => recipients.push(h.get_value().unwrap_or("".to_string())),
"cc" => recipients.push(h.get_value().unwrap_or("".to_string())),
"bcc" => recipients.push(h.get_value().unwrap_or("".to_string())),
let key = h.get_key();
match key.as_ref() {
"Subject" => subject = h.get_value(),
"From" => from = h.get_value(),
"To" => recipients.push(h.get_value()),
"cc" => recipients.push(h.get_value()),
"bcc" => recipients.push(h.get_value()),
"Received" | "Date" => {
if date == default_date {
let date_str = h.get_value();
match date_str {
Ok(date_str) => {
for ts in date_str.rsplit(';') {
date = match dateparse(ts) {
Ok(d) => d,
@ -148,15 +173,11 @@ impl Message {
break;
}
}
Err(_) => (),
}
}
}
_ => {}
}
}
}
let bodies = readmail::extract_body(&mut msg, false);
let bodies = readmail::extract_body(&msg, false);
Ok(Message {
body: bodies,
from,
@ -164,21 +185,29 @@ impl Message {
recipients,
date: date as u64,
id,
original: None,
original,
tags: HashSet::new(),
})
}
pub fn from_mailentry(mailentry: &mut MailEntry) -> Result<Self, MessageError> {
let hash = get_id(mailentry.data().expect("Unable to read the actual data"));
pub fn from_data(data: Vec<u8>) -> Result<Self, MessageError> {
let id = get_id(data.as_ref());
let parsed_mail = parse_mail(data.as_slice()).map_err(|_| MessageError {
message: String::from("Unable to parse email data"),
})?;
Self::from_parsedmail(&parsed_mail, id, data.clone())
}
pub fn from_mailentry(mailentry: MailEntry) -> Result<Self, MessageError> {
let id = mailentry.id();
mailentry.read_data().map_err(|e| MessageError {
message: format!("Failed to parse email id {}", id),
})?;
let data = mailentry.data().ok_or(MessageError {
message: format!("Mail {} could not read data", id),
})?;
match mailentry.parsed() {
Ok(mut msg) => Self::from_parsedmail(&mut msg, hash),
Ok(parsed) => Self::from_parsedmail(&parsed, String::from(id), data.clone()),
Err(e) => Err(MessageError {
message: format!(
"Failed on {}:{} -- MailEntryError: {}",
mailentry.id(),
hash,
e
),
message: format!("Failed to parse email id {}", id),
}),
}
}
@ -231,6 +260,10 @@ impl MessageBuilder {
self.recipients = Some(recipients.split(",").map(|s| String::from(s)).collect());
self
}
fn original(mut self, original: Vec<u8>) -> Self {
self.original = Some(original);
self
}
fn build(self) -> Message {
let msg = "Missing field for Message";
@ -244,7 +277,7 @@ impl MessageBuilder {
subject: self.subject.expect(msg),
recipients: self.recipients.expect(msg),
date: self.date.expect(msg),
original: None,
original: self.original.expect(msg),
tags: HashSet::new(),
}
}

88
src/readmail/mod.rs
View File

@ -1,13 +1,10 @@
use crate::message::{Body, Mime};
use html5ever::rcdom::{Handle, Node, NodeData, RcDom};
use html5ever::serialize::{serialize, SerializeOpts};
use html5ever::tendril::TendrilSink;
use html5ever::tree_builder::TreeBuilderOpts;
use html5ever::{local_name, parse_document, ParseOpts};
use log::{debug, error};
extern crate select;
use crate::message::{get_id, Body, Message, Mime};
use log::debug;
use mailparse::*;
use select::document::Document;
use select::predicate::Text;
use std::cmp::Ordering;
use std::time::{Duration, Instant};
fn cmp_body(x: &Body, y: &Body, prefer: &Mime) -> Ordering {
if x.mime == y.mime {
@ -20,20 +17,23 @@ fn cmp_body(x: &Body, y: &Body, prefer: &Mime) -> Ordering {
}
}
pub fn extract_body(msg: &mut ParsedMail, prefer_html: bool) -> Vec<Body> {
let mut raw_body = None;
pub fn extract_body(msg: &ParsedMail, prefer_html: bool) -> Vec<Body> {
let prefered_mime = if prefer_html {
Mime::Html
} else {
Mime::PlainText
};
if let Ok(text) = msg.get_body() {
let text = msg
.get_body()
.unwrap_or(msg.get_body_raw().map_or(String::from(""), |x| {
String::from_utf8(x).unwrap_or(String::from(""))
}));
let mime = Mime::from_str(&msg.ctype.mimetype);
raw_body = Some(Body::new(mime, String::from(text)));
};
let raw_body = Some(Body::new(mime, text));
let mut bodies = msg
.subparts
.iter_mut()
.iter()
.map(|mut s| {
let mime = Mime::from_str(&s.ctype.mimetype);
match mime {
@ -53,52 +53,24 @@ pub fn extract_body(msg: &mut ParsedMail, prefer_html: bool) -> Vec<Body> {
bodies.push(raw_body.expect("COULD NOT UNWRAP RAW_BODY"));
}
bodies.sort_unstable_by(|x, y| cmp_body(x, y, &prefered_mime));
if bodies.len() == 0 {
println!(
"No body for message: {}",
msg.headers
.iter()
.map(|x| format!("{}:{}", x.get_key(), x.get_value()))
.collect::<Vec<String>>()
.join("\n")
);
}
bodies
}
pub fn html2text(text: &str) -> String {
let opts = ParseOpts {
tree_builder: TreeBuilderOpts {
drop_doctype: true,
..Default::default()
},
..Default::default()
};
let dom = parse_document(RcDom::default(), opts)
.from_utf8()
.read_from(&mut text.as_bytes())
.expect("COULD NOT UNWRAP DOM");
let document_children = dom.document.children.borrow();
let html = document_children.get(0).expect("COULD NOT UNWRAP HTML");
let body_rc = html.children.borrow();
let body = body_rc
.iter()
.filter(|n| match n.data {
NodeData::Element { ref name, .. } => name.local == local_name!("body"),
_ => false,
})
.next();
let ret = match body {
Some(b) => render_tag(&b)
.into_iter()
.filter(|s| s != "")
.map(|s| s.trim().to_string())
.collect::<Vec<String>>()
.join("\n"),
None => "".to_string(),
};
ret
}
pub fn render_tag(node: &Handle) -> Vec<String> {
let mut ret = vec![];
match node.data {
NodeData::Text { ref contents } => ret.push(contents.borrow().trim().to_string()),
_ => {}
};
for child in node.children.borrow().iter() {
ret.append(&mut render_tag(child));
}
ret
let document = Document::from(text);
let text_nodes = document
.find(Text)
.map(|x| x.text())
.collect::<Vec<String>>();
return text_nodes.join("\n\n");
}

280
src/stores/_impl/rocksdb.rs
View File

@ -1,27 +1,38 @@
use crate::message::{Body, Message, Mime};
use crate::stores::{IMessageSearcher, IMessageStorage, MessageStoreError};
use crate::message::{Message, ShortMessage};
use crate::stores::{IMessageStorage, MessageStoreError};
use chrono::{DateTime, Utc};
use log::{info, trace, warn};
use rocksdb::{DBCompactionStyle, DBCompressionType};
use rocksdb::{DBVector, Options, DB};
use serde::{Deserialize, Serialize};
use std::cmp;
use serde_json::Result as SResult;
use std::collections::HashSet;
use std::path::PathBuf;
use std::path::Path;
use std::string::ToString;
const BYTES_IN_MB: usize = 1024 * 1024;
#[derive(Serialize, Deserialize)]
struct RocksDBMessage {
id: String,
body: String,
tag: HashSet<String>,
}
type RocksDBMessage = Message;
impl RocksDBMessage {
fn from(doc: DBVector) -> RocksDBMessage {
RocksDBMessage {
id: "a".to_string(),
body: "b".to_string(),
tag: HashSet::new(),
fn from_rocksdb(msg: DBVector) -> Result<RocksDBMessage, MessageStoreError> {
let msg_r = msg
.to_utf8()
.ok_or(Err(MessageStoreError::CouldNotGetMessage(
"Message is malformed in some way".to_string(),
)));
match msg_r {
Ok(msg) => serde_json::from_str(msg).map_err(|e| {
MessageStoreError::CouldNotGetMessage("Unable to parse the value".to_string())
}),
Err(e) => e,
}
}
fn to_rocksdb(&self) -> Result<(String, Vec<u8>), MessageStoreError> {
let id = self.id.clone();
let msg = serde_json::to_string(&self);
match msg {
Ok(msg) => Ok((id, msg.into_bytes())),
Err(e) => Err(MessageStoreError::CouldNotConvertMessage(format!(
"Failed to convert message for rocksdb: {}",
e
))),
}
}
}
@ -29,15 +40,36 @@ impl RocksDBMessage {
pub struct RocksDBStore {
db: DB,
}
impl IMessageStorage for RocksDBStore {
fn get_message(&self, id: String) -> Result<Message, MessageStoreError> {
self.get_message(id.as_str())
.ok_or(MessageStoreError::MessageNotFound(
"Unable to find message with that id".to_string(),
))
fn add_message(&mut self, msg: &RocksDBMessage) -> Result<String, MessageStoreError> {
let rocks_msg = msg.to_rocksdb();
match rocks_msg {
Ok((id, data)) => self
.db
.put(id.clone().into_bytes(), data)
.map_err(|_| {
MessageStoreError::CouldNotAddMessage("Failed to add message".to_string())
})
.map(|_| id),
Err(e) => Err(MessageStoreError::CouldNotAddMessage(format!(
"Failed to add message: {}",
e
))),
}
}
fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError> {
let m = self.db.get(id.into_bytes());
match m {
Ok(Some(message)) => Ok(Some(RocksDBMessage::from_rocksdb(message)?)),
Ok(None) => Err(MessageStoreError::CouldNotGetMessage(
"Message obtained was None".to_string(),
)),
Err(e) => Err(MessageStoreError::CouldNotGetMessage(format!(
"Could not get message due to : {}",
e
))),
}
fn add_message(&mut self, msg: Message) -> Result<String, MessageStoreError> {
unimplemented!();
}
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
unimplemented!()
@ -60,127 +92,105 @@ impl IMessageStorage for RocksDBStore {
unimplemented!()
}
}
impl IMessageSearcher for TantivyStore {
fn start_indexing_process(&mut self, num: usize) -> Result<(), MessageStoreError> {
if self.index_writer.is_none() {
let index_writer = self.get_index_writer(num)?;
self.index_writer = Some(index_writer);
}
Ok(())
}
fn finish_indexing_process(&mut self) -> Result<(), MessageStoreError> {
let writer = &mut self.index_writer;
match writer {
Some(writer) => match writer.commit() {
Ok(_) => Ok(()),
Err(e) => Err(MessageStoreError::CouldNotAddMessage(
"Failed to commit to index".to_string(),
)),
},
None => Err(MessageStoreError::CouldNotAddMessage(
"Trying to commit index without an actual index".to_string(),
)),
}
}
fn add_message(
&mut self,
msg: Message,
parsed_body: String,
) -> Result<String, MessageStoreError> {
self._add_message(msg, parsed_body)
}
fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError> {
Ok(self.fuzzy(query.as_str(), num))
}
fn search_by_date(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<Vec<Message>, MessageStoreError> {
Ok(vec![])
}
fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
Ok(())
}
fn tag_message_id(
&mut self,
id: String,
tags: HashSet<String>,
) -> Result<usize, MessageStoreError> {
let message = self.get_message(id.as_str());
match message {
Some(mut message) => {
let now = Instant::now();
self.start_indexing_process(1)?;
println!("{}", now.elapsed().as_nanos());
self._delete_message(&message)?;
println!("{}", now.elapsed().as_nanos());
message.tags = tags;
let body = message.get_body().clone();
self._add_message(message, body.value)?;
println!("{}", now.elapsed().as_nanos());
self.finish_indexing_process()?;
println!("{}", now.elapsed().as_nanos());
Ok(1)
}
None => Err(MessageStoreError::MessageNotFound(
"Could not tag message because the message was not found".to_string(),
)),
}
}
fn tag_message(
&mut self,
msg: Message,
tags: HashSet<String>,
) -> Result<usize, MessageStoreError> {
Ok(1)
}
fn get_messages_page(
&self,
start: Message,
num: usize,
) -> Result<Vec<Message>, MessageStoreError> {
Ok(vec![])
}
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
unimplemented!();
}
}
impl RocksDBStore {
pub fn new(path: PathBuf) -> Self {
unimplemented!()
pub fn new<P: AsRef<Path>>(path: P) -> Self {
let mut opts = Options::default();
opts.increase_parallelism(16);
opts.create_if_missing(true);
opts.set_compaction_style(DBCompactionStyle::Level);
opts.set_skip_stats_update_on_db_open(true);
opts.set_compression_type(DBCompressionType::Lz4);
opts.create_missing_column_families(true);
opts.set_use_direct_reads(true);
opts.set_allow_mmap_reads(true);
opts.set_allow_mmap_writes(true);
opts.set_max_open_files(2);
let db = DB::open_default(path).unwrap();
RocksDBStore { db }
}
fn _add_message(
&mut self,
msg: Message,
parsed_body: String,
_msg: &Message,
_parsed_body: String,
) -> Result<String, MessageStoreError> {
unimplemented!()
}
fn _delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError> {
unimplemented!()
}
pub fn tag_doc(&self, doc: Document, tags: Vec<String>) -> Result<(), MessageStoreError> {
fn _delete_message(&mut self, _msg: &Message) -> Result<(), MessageStoreError> {
unimplemented!()
}
pub fn latest(&self, num: usize) -> Vec<Message> {
pub fn latest(&self, _num: usize) -> Vec<Message> {
unimplemented!()
}
pub fn by_date(&self) {
unimplemented!();
}
pub fn get_doc(&self, id: &str) -> Result<Document, tantivy::Error> {
unimplemented!();
}
pub fn get_message(&self, id: &str) -> Option<Message> {
unimplemented!();
}
pub fn search(&self, text: &str, num: usize) -> Vec<Message> {
unimplemented!();
}
}
#[cfg(test)]
mod test {
use super::RocksDBStore;
use crate::message::{Body, Message, Mime};
use crate::stores::IMessageStorage;
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use rocksdb::{Options, DB};
use std::collections::HashSet;
struct StoreInit {
path: Option<String>,
store: RocksDBStore,
}
impl StoreInit {
fn new() -> StoreInit {
let rand_string: String = thread_rng().sample_iter(&Alphanumeric).take(5).collect();
let mut path = std::path::PathBuf::new();
path.push("./test_db/");
path.push(rand_string);
let newdb = RocksDBStore::new(&path);
StoreInit {
path: path.to_str().map(|s| s.to_string()),
store: newdb,
}
}
}
impl Drop for StoreInit {
fn drop(&mut self) {
let opts = Options::default();
let path = self.path.as_ref().unwrap();
DB::destroy(&opts, path);
std::fs::remove_dir_all(path);
}
}
#[test]
fn add_message() {
let store = &mut StoreInit::new().store;
let message = Message {
id: "some_id".to_string(),
from: "It's me, Mario!".to_string(),
body: vec![Body {
mime: Mime::PlainText,
value: "Test body".to_string(),
}],
subject: "test_subject".to_string(),
recipients: vec!["r1".to_string(), "r2".to_string()],
date: 4121251,
original: vec![0],
tags: vec!["tag1".to_string(), "tag2".to_string()]
.into_iter()
.collect::<HashSet<String>>(),
};
store.add_message(&message).ok().unwrap();
let retrieved = store
.get_message("some_id".to_string())
.ok()
.unwrap()
.unwrap();
assert_eq!(message, retrieved);
}
#[test]
fn test_rocksdb2() {
let store = &StoreInit::new().store;
store.db.put(b"key", b"value2").unwrap();
let get = store.db.get(b"key").ok().unwrap().unwrap();
assert_eq!("value2", get.to_utf8().unwrap());
}
}

282
src/stores/_impl/tantivy.rs
View File

@ -1,13 +1,11 @@
use crate::stores::{IMessageSearcher, IMessageStorage, MessageStoreError};
use chrono::{DateTime, Utc};
use log::{debug, error, info, trace, warn};
use log::info;
use std::cmp;
use std::collections::HashSet;
use std::fs;
use std::panic;
use std::sync::mpsc;
use std::time::{Duration, Instant};
use std::time::{SystemTime, UNIX_EPOCH};
use std::time::Instant;
use crate::message::{Body, Message, Mime};
use std::path::PathBuf;
@ -17,18 +15,22 @@ use tantivy::collector::{Count, TopDocs};
use tantivy::directory::MmapDirectory;
use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, RangeQuery, TermQuery};
use tantivy::schema::*;
use tantivy::DocAddress;
const BYTES_IN_MB: usize = 1024 * 1024;
pub type TantivyMessage = Message;
impl TantivyMessage {
fn from(doc: Document, schema: &EmailSchema) -> TantivyMessage {
let original = match doc.get_first(schema.original) {
Some(t) => match t.text() {
Some(t) => Some(String::from(t)),
None => None,
},
None => None,
pub trait TantivyFrom<T> {
fn from_tantivy(doc: Document, schema: &EmailSchema) -> T;
}
impl TantivyFrom<TantivyMessage> for TantivyMessage {
fn from_tantivy(doc: Document, schema: &EmailSchema) -> TantivyMessage {
let original: Result<Vec<u8>, _> = match doc
.get_first(schema.original)
.expect("Unable to get original message")
{
Value::Bytes(b) => Ok(b.clone()),
_ => Err("Missing original email from the index"),
};
let tags: HashSet<String> = doc
@ -37,7 +39,6 @@ impl TantivyMessage {
.filter_map(|s| s.text())
.map(|s| String::from(s))
.collect();
TantivyMessage {
id: doc
.get_first(schema.id)
@ -45,12 +46,29 @@ impl TantivyMessage {
.text()
.expect("Message ID is always a string")
.to_string(),
from: String::from(
doc.get_first(schema.from)
.expect("Message without from")
.text()
.expect("Message with non-text from"),
),
subject: String::from(
doc.get_first(schema.subject)
.expect("Message without subject")
.text()
.expect("Message with non-text subject"),
),
date: doc
.get_first(schema.date)
.map_or(0, |v: &tantivy::schema::Value| v.u64_value()),
recipients: doc
.get_first(schema.recipients)
.unwrap_or(&tantivy::schema::Value::Str(String::from("a")))
.text()
.expect("Message with non-text recipients")
.split(",")
.map(|s| String::from(s))
.collect(),
body: vec![Body {
mime: Mime::PlainText,
value: String::from(
@ -60,30 +78,14 @@ impl TantivyMessage {
.expect("Message with non-text body"),
),
}],
from: String::from(
doc.get_first(schema.from)
.expect("Message without from")
.text()
.expect("Message with non-text from"),
),
recipients: vec![doc
.get_first(schema.recipients)
.expect("Message without recipients")
.text()
.expect("Message with non-text recipients")
.split(",")
.map(|s| String::from(s))
.collect()],
date: doc
.get_first(schema.date)
.map_or(0, |v: &tantivy::schema::Value| v.u64_value()),
original,
original: original.expect("Original was missing from the index"),
tags,
}
}
}
struct EmailSchema {
pub struct EmailSchema {
schema: Schema,
subject: Field,
body: Field,
@ -92,8 +94,8 @@ struct EmailSchema {
thread: Field,
id: Field,
date: Field,
original: Field,
tag: Field,
original: Field,
}
impl Default for EmailSchema {
@ -103,15 +105,15 @@ impl Default for EmailSchema {
let body = schema_builder.add_text_field("body", TEXT | STORED);
let from = schema_builder.add_text_field("from", TEXT | STORED);
let recipients = schema_builder.add_text_field("recipients", TEXT | STORED);
let thread = schema_builder.add_text_field("thread", STRING | STORED);
let thread = schema_builder.add_text_field("thread", STRING);
let id = schema_builder.add_text_field("id", STRING | STORED);
let tag = schema_builder.add_text_field("tag", STRING | STORED);
let original = schema_builder.add_text_field("original", STORED);
let dateoptions = IntOptions::default()
.set_fast(Cardinality::SingleValue)
.set_stored()
.set_indexed();
let date = schema_builder.add_u64_field("date", dateoptions);
let original = schema_builder.add_text_field("original", TEXT | STORED);
let schema = schema_builder.build();
EmailSchema {
schema,
@ -122,13 +124,13 @@ impl Default for EmailSchema {
thread,
id,
date,
original,
tag,
original,
}
}
}
impl EmailSchema {
pub fn new() -> EmailSchema {
pub fn _new() -> EmailSchema {
EmailSchema::default()
}
}
@ -136,52 +138,22 @@ impl EmailSchema {
pub struct TantivyStore {
email: EmailSchema,
index: tantivy::Index,
index_writer: Option<tantivy::IndexWriter>,
reader: tantivy::IndexReader,
writer: Option<tantivy::IndexWriter>,
threads: Option<usize>,
mem_per_thread: Option<usize>,
}
impl IMessageStorage for TantivyStore {
fn get_message(&self, id: String) -> Result<Message, MessageStoreError> {
self.get_message(id.as_str())
.ok_or(MessageStoreError::MessageNotFound(
"Unable to find message with that id".to_string(),
))
}
fn add_message(&mut self, msg: Message) -> Result<String, MessageStoreError> {
unimplemented!();
}
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
unimplemented!()
}
fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
unimplemented!()
}
fn get_messages_page(
&self,
start: usize,
num: usize,
) -> Result<Vec<Message>, MessageStoreError> {
Ok(self.latest(num))
}
fn get_by_date(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<Vec<Message>, MessageStoreError> {
unimplemented!()
}
}
impl IMessageSearcher for TantivyStore {
fn start_indexing_process(&mut self, num: usize) -> Result<(), MessageStoreError> {
if self.index_writer.is_none() {
let index_writer = self.get_index_writer(num)?;
self.index_writer = Some(index_writer);
if self.writer.is_none() {
let writer = self.get_index_writer(num)?;
self.writer = Some(writer);
}
Ok(())
}
fn finish_indexing_process(&mut self) -> Result<(), MessageStoreError> {
let writer = &mut self.index_writer;
let writer = &mut self.writer;
match writer {
Some(writer) => match writer.commit() {
Ok(_) => Ok(()),
@ -202,62 +174,33 @@ impl IMessageSearcher for TantivyStore {
) -> Result<String, MessageStoreError> {
self._add_message(msg, parsed_body)
}
fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError> {
Ok(self.fuzzy(query.as_str(), num))
}
fn search_by_date(
fn search_fuzzy(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<Vec<Message>, MessageStoreError> {
Ok(vec![])
}
fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
Ok(())
}
fn tag_message_id(
&mut self,
id: String,
tags: HashSet<String>,
) -> Result<usize, MessageStoreError> {
let message = self.get_message(id.as_str());
match message {
Some(mut message) => {
let now = Instant::now();
self.start_indexing_process(1)?;
println!("{}", now.elapsed().as_nanos());
self._delete_message(&message)?;
println!("{}", now.elapsed().as_nanos());
message.tags = tags;
let body = message.get_body().clone();
self._add_message(message, body.value)?;
println!("{}", now.elapsed().as_nanos());
self.finish_indexing_process()?;
println!("{}", now.elapsed().as_nanos());
Ok(1)
}
None => Err(MessageStoreError::MessageNotFound(
"Could not tag message because the message was not found".to_string(),
)),
}
}
fn tag_message(
&mut self,
msg: Message,
tags: HashSet<String>,
) -> Result<usize, MessageStoreError> {
Ok(1)
}
fn get_messages_page(
&self,
start: Message,
query: String,
num: usize,
) -> Result<Vec<Message>, MessageStoreError> {
Ok(vec![])
) -> Result<Vec<TantivyMessage>, MessageStoreError> {
Ok(self.search(query.as_str(), num))
}
fn delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError> {
Ok(())
}
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
unimplemented!();
}
fn latest(&mut self, num: usize) -> Result<Vec<Message>, MessageStoreError> {
self._latest(num, None)
}
fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError> {
Ok(self._get_message(id.as_ref()))
}
fn get_messages_page(
&self,
start: usize,
num: usize,
) -> Result<Vec<Message>, MessageStoreError> {
self._latest(num, Some(start))
}
}
impl TantivyStore {
@ -266,9 +209,14 @@ impl TantivyStore {
}
fn _new(path: PathBuf, ro: bool) -> Self {
let email = EmailSchema::default();
let index = TantivyStore::open_or_create_index(path, email.schema.clone());
let reader = index
.reader()
.expect("Unable to create an index reader for this index. Is the index corrupted?");
TantivyStore {
index: TantivyStore::open_or_create_index(path, email.schema.clone()),
index_writer: None,
index,
reader,
writer: None,
email,
threads: None,
mem_per_thread: None,
@ -298,7 +246,7 @@ impl TantivyStore {
msg: Message,
parsed_body: String,
) -> Result<String, MessageStoreError> {
let writer = &mut self.index_writer;
let writer = &mut self.writer;
match writer {
Some(indexer) => {
let mut document = Document::new();
@ -308,12 +256,13 @@ impl TantivyStore {
document.add_text(email.body, parsed_body.as_str());
document.add_text(email.from, msg.from.as_str());
document.add_text(email.recipients, msg.recipients.join(", ").as_str());
document.add_bytes(email.original, msg.original);
document.add_u64(email.date, msg.date);
msg.tags
.into_iter()
.iter()
.for_each(|t| document.add_text(email.tag, t.as_str()));
indexer.add_document(document);
Ok(msg.id)
Ok(msg.id.clone())
}
None => Err(MessageStoreError::CouldNotAddMessage(
"No indexer was allocated".to_string(),
@ -321,7 +270,7 @@ impl TantivyStore {
}
}
fn _delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError> {
let writer = &mut self.index_writer;
let writer = &mut self.writer;
match writer {
Some(indexer) => {
let term = Term::from_field_text(self.email.id, msg.id.as_ref());
@ -333,13 +282,13 @@ impl TantivyStore {
)),
}
}
pub fn tag_doc(&self, doc: Document, tags: Vec<String>) -> Result<(), tantivy::TantivyError> {
let mut index_writer = self.get_index_writer(1).ok().unwrap();
let id = TantivyMessage::from(doc, &self.email).id;
pub fn _tag_doc(&self, doc: Document, tags: Vec<String>) -> Result<(), tantivy::TantivyError> {
let mut writer = self.get_index_writer(1).ok().unwrap();
let id = TantivyMessage::from_tantivy(doc, &self.email).id;
let term = Term::from_field_text(self.email.id, id.as_ref());
index_writer.delete_term(term.clone());
index_writer.commit()?;
self.index.load_searchers()
writer.delete_term(term.clone());
writer.commit()?;
self.reader.reload()
}
fn get_index_writer(
@ -377,7 +326,7 @@ impl TantivyStore {
}
};
info!(
"For your information, we're using {} threads with {}mb memory per thread",
"We're using {} threads with {}mb memory per thread",
num_cpu,
mem_per_thread / BYTES_IN_MB
);
@ -392,30 +341,33 @@ impl TantivyStore {
}
}
pub fn latest(&self, num: usize) -> Vec<TantivyMessage> {
let searcher = self.index.searcher();
let docs = searcher
pub fn _latest(
&self,
num: usize,
_skip: Option<usize>,
) -> Result<Vec<TantivyMessage>, MessageStoreError> {
let searcher = self.reader.searcher();
let skip = _skip.unwrap_or(0);
let mut docs = searcher
.search(
&AllQuery,
&TopDocs::with_limit(num).order_by_field::<u64>(self.email.date),
&TopDocs::with_limit(num + skip).order_by_u64_field(self.email.date),
)
.unwrap();
.map_err(|e| MessageStoreError::CouldNotGetMessages(vec![]))?;
let mut ret = vec![];
for doc in docs {
let page = docs
.drain(skip..)
.collect::<Vec<(u64, tantivy::DocAddress)>>();
for doc in page {
let retrieved_doc = searcher.doc(doc.1).unwrap();
ret.push(TantivyMessage::from(retrieved_doc, &self.email));
ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
}
ret
Ok(ret)
}
pub fn by_date(&self) {
let searcher = self.index.searcher();
let docs = RangeQuery::new_u64(self.email.date, 1522704682..1524704682);
let numdocs = searcher.search(&docs, &Count).unwrap();
}
pub fn get_doc(&self, id: &str) -> Result<Document, tantivy::Error> {
self.index.load_searchers()?;
let searcher = self.index.searcher();
pub fn get_doc(&self, id: &str) -> Result<Document, tantivy::TantivyError> {
// Is this needed? self.reader.load_searchers()?;
let searcher = self.reader.searcher();
let termq = TermQuery::new(
Term::from_field_text(self.email.id, id.as_ref()),
IndexRecordOption::Basic,
@ -424,7 +376,7 @@ impl TantivyStore {
match addr {
Ok(doc) => match doc.first() {
Some((_score, doc_address)) => searcher.doc(*doc_address),
None => Err(tantivy::Error::InvalidArgument(
None => Err(tantivy::TantivyError::InvalidArgument(
"Document not found".to_string(),
)),
},
@ -432,27 +384,24 @@ impl TantivyStore {
Err(e) => Err(e),
}
}
pub fn get_message(&self, id: &str) -> Option<TantivyMessage> {
pub fn _get_message(&self, id: &str) -> Option<TantivyMessage> {
let doc = self.get_doc(id);
match doc {
Ok(doc) => Some(TantivyMessage::from(doc, &self.email)),
Ok(doc) => Some(TantivyMessage::from_tantivy(doc, &self.email)),
Err(_) => None,
}
}
pub fn search(&self, text: &str, num: usize) -> Vec<TantivyMessage> {
let searcher = self.index.searcher();
let searcher = self.reader.searcher();
let term = Term::from_field_text(self.email.subject, text);
let term_body = Term::from_field_text(self.email.body, text);
let query = TermQuery::new(term, IndexRecordOption::Basic);
let query_body = TermQuery::new(term_body, IndexRecordOption::Basic);
let top_docs_by_date = TopDocs::with_limit(num).order_by_field::<u64>(self.email.date);
let queries: Vec<(Occur, Box<Query>)> = vec![];
let bquery = BooleanQuery::from(queries);
let top_docs_by_date = TopDocs::with_limit(num).order_by_u64_field(self.email.date);
let bquery = BooleanQuery::new_multiterms_query(vec![term, term_body]);
let top_docs = searcher.search(&bquery, &top_docs_by_date).unwrap();
let mut ret = vec![];
for doc in top_docs {
let retrieved_doc = searcher.doc(doc.1).unwrap();
ret.push(TantivyMessage::from(retrieved_doc, &self.email));
ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
}
ret
}
@ -460,7 +409,7 @@ impl TantivyStore {
pub fn fuzzy(&self, text: &str, num: usize) -> Vec<TantivyMessage> {
let mut terms = text.split(' ').collect::<Vec<&str>>();
terms.insert(0, text);
let searcher = self.index.searcher();
let searcher = self.reader.searcher();
let mut ret = self.search(text, num);
for n in 1..2 {
if ret.len() < num {
@ -473,13 +422,12 @@ impl TantivyStore {
queries.push((Occur::Should, Box::new(query)));
queries.push((Occur::Should, Box::new(query_body)));
}
let top_docs_by_date =
TopDocs::with_limit(num).order_by_field::<u64>(self.email.date);
let top_docs_by_date = TopDocs::with_limit(num).order_by_u64_field(self.email.date);
let bquery = BooleanQuery::from(queries);
let top_docs = searcher.search(&bquery, &top_docs_by_date).unwrap();
for doc in top_docs {
let retrieved_doc = searcher.doc(doc.1).unwrap();
ret.push(TantivyMessage::from(retrieved_doc, &self.email));
ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
}
}
}

52
src/stores/message_store.rs
View File

@ -1,7 +1,7 @@
use crate::message::{get_id, Body, Message, Mime};
use crate::message::{Message, ShortMessage};
use crate::stores::{IMessageSearcher, IMessageStorage, IMessageStore, MessageStoreError};
use chrono::{DateTime, Utc};
use log::{debug, error, info};
use log::error;
use maildir::{MailEntry, Maildir};
use pbr::{MultiBar, ProgressBar};
use rayon::prelude::*;
@ -9,25 +9,17 @@ use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use std::time::{Duration, Instant};
pub struct MessageStore<I, S>
where
I: IMessageSearcher,
S: IMessageStorage,
{
pub searcher: Box<I>,
pub storage: Box<S>,
pub struct MessageStore {
pub searcher: Box<dyn IMessageSearcher>,
pub storage: Option<Box<dyn IMessageStorage>>,
progress: Option<ProgressBar<pbr::Pipe>>,
display_progress: bool,
}
impl<I, S> IMessageStore for MessageStore<I, S>
where
I: IMessageSearcher,
S: IMessageStorage,
{
fn get_message(&self, id: String) -> Result<Message, MessageStoreError> {
self.storage.get_message(id)
impl IMessageStore for MessageStore {
fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError> {
self.searcher.get_message(id)
}
fn add_message(
@ -35,7 +27,9 @@ where
msg: Message,
parsed_body: String,
) -> Result<String, MessageStoreError> {
self.searcher.add_message(msg, parsed_body)
let id = msg.id.clone();
self.searcher.add_message(msg, parsed_body)?;
Ok(id)
}
fn add_maildir(&mut self, path: PathBuf, all: bool) -> Result<usize, MessageStoreError> {
@ -46,7 +40,7 @@ where
id: String,
tags: HashSet<String>,
) -> Result<usize, MessageStoreError> {
self.searcher.tag_message_id(id, tags)
unimplemented!();
}
fn tag_message(
@ -65,7 +59,7 @@ where
start: usize,
num: usize,
) -> Result<Vec<Message>, MessageStoreError> {
self.storage.get_messages_page(start, num)
self.searcher.get_messages_page(start, num)
}
fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError> {
@ -82,12 +76,12 @@ where
unimplemented!();
}
}
impl<I, S> MessageStore<I, S>
where
I: IMessageSearcher,
S: IMessageStorage,
{
pub fn new(searcher: Box<I>, storage: Box<S>, display_progress: bool) -> Self {
impl MessageStore {
pub fn new(
searcher: Box<dyn IMessageSearcher>,
storage: Option<Box<dyn IMessageStorage>>,
display_progress: bool,
) -> Self {
MessageStore {
searcher,
storage,
@ -119,7 +113,7 @@ where
let mut mb = MultiBar::new();
mb.println(&format!("Indexing {} emails", num));
let mut index_bar = mb.create_bar(num as u64);
if num < 10000000 {
if num < 10_000_000 {
mb.println("This will take no time!");
}
index_bar.message("Indexed ");
@ -155,8 +149,8 @@ where
mails
.into_par_iter()
.for_each_with(tx, |tx, msg| match msg {
Ok(mut unparsed_msg) => {
let message = Message::from_mailentry(&mut unparsed_msg);
Ok(unparsed_msg) => {
let message = Message::from_mailentry(unparsed_msg);
match message {
Ok(msg) => {
let parsed_body = msg.get_body().as_text();

102
src/stores/mod.rs
View File

@ -6,15 +6,16 @@ mod _impl;
use _impl::rocksdb;
use _impl::tantivy;
use std::fmt;
mod message_store;
pub mod message_store;
use message_store::MessageStore;
use std::time::Instant;
pub enum Searchers {
Tantivy(PathBuf),
}
pub enum Storages {
Tantivy(PathBuf),
Rocksdb(PathBuf),
Tantivy(PathBuf),
}
pub struct MessageStoreBuilder {
@ -51,7 +52,7 @@ impl MessageStoreBuilder {
}
}
pub fn new_from_cfg(pathbuf: PathBuf) -> MessageStoreBuilder {
pub fn new_from_cfg(_pathbuf: PathBuf) -> MessageStoreBuilder {
unimplemented!();
}
@ -74,24 +75,31 @@ impl MessageStoreBuilder {
self.storage = Some(storage);
self
}
pub fn maildir(&mut self, path: PathBuf) -> &mut Self {
self.maildir_path = Some(path);
self
}
pub fn build(&self) -> Result<Box<IMessageStore>, MessageStoreBuilderError> {
let store = match &self.storage {
None => Err(MessageStoreBuilderError::CouldNotCreateStoreError(
"No store type was provided".to_string(),
)),
Some(store_type) => match store_type {
Storages::Tantivy(path) => match self.read_only {
true => Ok(tantivy::TantivyStore::new_ro(std::path::PathBuf::from(
path,
))),
false => Ok(tantivy::TantivyStore::new(std::path::PathBuf::from(path))),
},
Storages::Rocksdb => Err(MessageStoreBuilderError::CouldNotCreateStoreError(
"Rocksdb is not yet supported, try again later".to_string(),
)),
},
}?;
pub fn build(&self) -> Result<impl IMessageStore, MessageStoreBuilderError> {
//let storage: Result<Option<Box<dyn IMessageStorage>>, MessageStoreBuilderError> =
// match &self.storage {
// None => Ok(None),
// Some(store_type) => match store_type {
// Storages::Rocksdb(path) => {
// let mut p = path.clone();
// p.push("storage");
// Ok(Some(Box::new(rocksdb::RocksDBStore::new(p))))
// }
// Storages::Tantivy(path) => {
// let mut p = path.clone();
// p.push("storage");
// tantivy = Some(tantivy::TantivyStore::new(
// std::path::PathBuf::from(p),
// ));
// Ok(Box::new(tantivy))
// }
// };
// },
let searcher = match &self.searcher {
None => Err(MessageStoreBuilderError::CouldNotCreateStoreError(
@ -99,17 +107,14 @@ impl MessageStoreBuilder {
)),
Some(searcher_type) => match searcher_type {
Searchers::Tantivy(path) => {
Ok(tantivy::TantivyStore::new(std::path::PathBuf::from(path)))
let mut p = path.clone();
p.push("searcher");
Ok(tantivy::TantivyStore::new(std::path::PathBuf::from(p)))
}
},
}?;
};
Ok(Box::new(MessageStore::<
tantivy::TantivyStore,
tantivy::TantivyStore,
>::new(
Box::new(searcher), Box::new(store), !self.debug
)))
Ok(MessageStore::new(Box::new(searcher?), None, !self.debug))
}
}
@ -118,7 +123,9 @@ pub enum MessageStoreError {
CouldNotAddMessage(String),
CouldNotOpenMaildir(String),
CouldNotModifyMessage(String),
CouldNotGetMessage(String),
CouldNotGetMessages(Vec<String>),
CouldNotConvertMessage(String),
InvalidQuery(String),
}
@ -134,7 +141,11 @@ impl fmt::Display for MessageStoreError {
MessageStoreError::CouldNotGetMessages(s) => {
format!("Could not get messages {}", s.join(", "))
}
MessageStoreError::CouldNotGetMessage(s) => format!("Could not get message {}", s),
MessageStoreError::InvalidQuery(s) => format!("Could query message {}", s),
MessageStoreError::CouldNotConvertMessage(s) => {
format!("Could not convert message {}", s)
}
};
write!(f, "Message Store Error {}", msg)
}
@ -147,37 +158,24 @@ pub trait IMessageSearcher {
parsed_body: String,
) -> Result<String, MessageStoreError>;
fn search_fuzzy(&self, query: String, num: usize) -> Result<Vec<Message>, MessageStoreError>;
fn search_by_date(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<Vec<Message>, MessageStoreError>;
fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError>;
fn tag_message_id(
&mut self,
id: String,
tags: HashSet<String>,
) -> Result<usize, MessageStoreError>;
fn tag_message(
&mut self,
msg: Message,
tags: HashSet<String>,
) -> Result<usize, MessageStoreError>;
fn delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError>;
fn get_messages_page(
&self,
start: Message,
num: usize,
) -> Result<Vec<Message>, MessageStoreError>;
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError>;
fn start_indexing_process(&mut self, num: usize) -> Result<(), MessageStoreError>;
fn finish_indexing_process(&mut self) -> Result<(), MessageStoreError>;
fn latest(&mut self, num: usize) -> Result<Vec<Message>, MessageStoreError>;
fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError>;
fn get_messages_page(
&self,
start: usize,
num: usize,
) -> Result<Vec<Message>, MessageStoreError>;
}
pub trait IMessageStorage {
fn get_message(&self, id: String) -> Result<Message, MessageStoreError>;
fn add_message(&mut self, msg: Message) -> Result<String, MessageStoreError>;
fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError>;
fn add_message(&mut self, msg: &Message) -> Result<String, MessageStoreError>;
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError>;
fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError>;
fn get_messages_page(
@ -193,7 +191,7 @@ pub trait IMessageStorage {
}
pub trait IMessageStore {
fn get_message(&self, id: String) -> Result<Message, MessageStoreError>;
fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError>;
fn add_message(
&mut self,
msg: Message,

4
src/terminal/events.rs
View File

@ -15,8 +15,8 @@ pub enum Event<I> {
/// type is handled in its own thread and returned to a common `Receiver`
pub struct Events {
rx: mpsc::Receiver<Event<Key>>,
input_handle: thread::JoinHandle<()>,
tick_handle: thread::JoinHandle<()>,
pub input_handle: thread::JoinHandle<()>,
pub tick_handle: thread::JoinHandle<()>,
}
#[derive(Debug, Clone, Copy)]

6
src/terminal/input/mod.rs
View File

@ -6,15 +6,15 @@ mod reader;
mod search;
pub struct InputHandler {
name: String,
pub name: String,
pre: bool,
f: Box<Runnable>,
children: Vec<Box<InputHandler>>,
}
#[derive(Debug)]
pub struct NoopRunner {}
impl Runnable for NoopRunner {
pub struct _NoopRunner {}
impl Runnable for _NoopRunner {
fn run(&self, _e: &Event<Key>, _store: &mut Store) -> bool {
false
}

22
src/terminal/input/reader.rs
View File

@ -12,39 +12,35 @@ impl Runnable for ReaderRunner {
Event::Input(key) => match key {
Key::Esc | Key::Char('q') => {
store.reader_store.read(None);
return true;
true
}
Key::Char('j') | Key::Down => {
store.reader_store.scroll(3);
return true;
true
}
Key::Char('k') | Key::Up => {
store.reader_store.scroll(-3);
return true;
true
}
Key::Ctrl('u') | Key::PageUp => {
store.reader_store.scroll(-20);
return true;
true
}
Key::Ctrl('d') | Key::PageDown => {
store.reader_store.scroll(20);
return true;
true
}
Key::Char('t') => {
store.tags_store.edit(store.reader_store.get_message());
return true;
true
}
Key::Home => {
store.reader_store.scroll_top();
return true;
}
_ => {
return false;
true
}
_ => false,
},
_ => {
return false;
}
_ => false,
}
} else {
false

6
src/terminal/mod.rs
View File

@ -5,10 +5,8 @@ mod views;
use crate::stores::{MessageStoreBuilder, Searchers, Storages};
use events::Events;
use input::{handlers, run};
use std::cell::RefCell;
use std::io;
use std::path::PathBuf;
use std::rc::Rc;
use store::Store;
use termion::raw::IntoRawMode;
use tui::backend::TermionBackend;
@ -38,10 +36,10 @@ pub fn start(index: PathBuf) -> Result<(), io::Error> {
break;
};
}
terminal.clear();
terminal.clear()?;
}
Err(e) => {
terminal.clear();
terminal.clear()?;
println!("Error {}", e);
}
};

10
src/terminal/store/list.rs
View File

@ -7,11 +7,11 @@ pub struct ListStore<'a> {
pub page_size: usize,
pub curr_idx: usize,
pub fetched_first: bool,
pub message_store: &'a Box<IMessageStore>,
pub message_store: &'a IMessageStore,
}
impl<'a> ListStore<'a> {
pub fn new(msg_store: &'a Box<IMessageStore>) -> ListStore<'a> {
pub fn new(msg_store: &'a IMessageStore) -> ListStore<'a> {
ListStore {
messages: vec![],
selected: 0,
@ -22,12 +22,6 @@ impl<'a> ListStore<'a> {
}
}
pub fn set_results(&mut self, messages: Vec<Message>) -> &Self {
self.messages = messages;
self.set_selected(0);
self
}
pub fn get_selected(&mut self) -> Option<&Message> {
self.messages.get(self.selected)
}

6
src/terminal/store/mod.rs
View File

@ -12,16 +12,16 @@ pub struct Store<'a> {
pub exit: bool,
pub list_store: ListStore<'a>,
pub search_store: SearchStore<'a>,
pub reader_store: ReaderStore,
pub reader_store: ReaderStore<'a>,
pub tags_store: TagsStore<'a>,
}
impl<'a> Store<'a> {
pub fn new(message_store: &'a Box<IMessageStore>) -> Store {
pub fn new(message_store: &'a IMessageStore) -> Store {
Store {
exit: false,
search_store: SearchStore::new(message_store),
list_store: ListStore::new(message_store),
reader_store: ReaderStore::new(),
reader_store: ReaderStore::new(message_store),
tags_store: TagsStore::new(message_store),
}
}

17
src/terminal/store/reader.rs
View File

@ -1,16 +1,19 @@
use crate::message::Message;
use crate::stores::IMessageStore;
use std::cmp::max;
pub struct ReaderStore {
pub struct ReaderStore<'a> {
pub message: Option<Message>,
pub scroll: u16,
pub storage: &'a IMessageStore,
}
impl ReaderStore {
pub fn new() -> ReaderStore {
impl<'a> ReaderStore<'a> {
pub fn new(storage: &'a IMessageStore) -> ReaderStore<'a> {
ReaderStore {
message: None,
scroll: 0,
storage,
}
}
@ -21,9 +24,15 @@ impl ReaderStore {
}
}
pub fn read(&mut self, msg: Option<&Message>) {
self.message = msg.cloned();
let msg = msg.cloned();
match msg {
Some(msg) => {
self.message = self.storage.get_message(msg.id).ok().unwrap();
self.scroll = 0;
}
None => self.message = None,
}
}
pub fn scroll_top(&mut self) {
self.scroll = 0;

4
src/terminal/store/search.rs
View File

@ -4,11 +4,11 @@ use crate::stores::IMessageStore;
pub struct SearchStore<'a> {
pub search_term: String,
pub searching: bool,
pub searcher: &'a Box<IMessageStore>,
pub searcher: &'a IMessageStore,
pub results: Vec<Message>,
}
impl<'a> SearchStore<'a> {
pub fn new(msg_store: &'a Box<IMessageStore>) -> SearchStore {
pub fn new(msg_store: &'a IMessageStore) -> SearchStore {
SearchStore {
search_term: String::from(""),
searching: false,

4
src/terminal/store/tags.rs
View File

@ -2,11 +2,11 @@ use crate::message::Message;
use crate::stores::IMessageStore;
pub struct TagsStore<'a> {
pub message_store: &'a Box<IMessageStore>,
pub message_store: &'a IMessageStore,
pub message: Option<Message>,
}
impl<'a> TagsStore<'a> {
pub fn new(msg_store: &'a Box<IMessageStore>) -> TagsStore<'a> {
pub fn new(msg_store: &'a IMessageStore) -> TagsStore<'a> {
TagsStore {
message: None,
message_store: msg_store,

5
src/terminal/views/email_read.rs
View File

@ -1,7 +1,6 @@
use crate::message::Message;
use tui::backend::Backend;
use tui::layout::{Alignment, Rect};
use tui::layout::{Constraint, Direction, Layout};
use tui::layout::Rect;
use tui::style::{Modifier, Style};
use tui::widgets::{Block, Borders, Paragraph, Text, Widget};
use tui::Frame;
@ -19,7 +18,7 @@ pub fn draw<B: Backend>(f: &mut Frame<B>, message: &Message, scroll: u16) {
let block = Block::default()
.borders(Borders::ALL)
.title_style(Style::default().modifier(Modifier::Bold));
.title_style(Style::default().modifier(Modifier::BOLD));
Paragraph::new(text.iter())
.block(block.clone())
.wrap(true)

9
src/terminal/views/search_results.rs
View File

@ -1,9 +1,8 @@
use crate::terminal::store::Store;
use std::io;
use tui::backend::{Backend, TermionBackend};
use tui::layout::{Constraint, Direction, Layout, Rect};
use tui::backend::Backend;
use tui::layout::Rect;
use tui::style::{Color, Modifier, Style};
use tui::widgets::{Block, Borders, List, SelectableList, Text, Widget};
use tui::widgets::{Block, Borders, SelectableList, Widget};
use tui::Frame;
pub fn draw<B: Backend>(f: &mut Frame<B>, area: Rect, store: &Store) {
@ -23,7 +22,7 @@ pub fn draw<B: Backend>(f: &mut Frame<B>, area: Rect, store: &Store) {
)
.select(Some(store.list_store.selected))
.style(style)
.highlight_style(style.fg(Color::LightGreen).modifier(Modifier::Bold))
.highlight_style(style.fg(Color::LightGreen).modifier(Modifier::BOLD))
.highlight_symbol(">")
.render(f, area);
}