rms/rms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Partial cleanup

Browse Source
This commit is contained in:
Lewis Diamond 2020-09-01 16:20:50 -04:00
parent f74963cf3c
commit 01a42cebf6
23 changed files with 629 additions and 734 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

772
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

22
Cargo.toml
View File

@ -7,15 +7,17 @@ edition = "2018"
[dependencies]
#maildir = { git = "https://github.com/lewisdiamond/maildir.git" }
maildir = { path = "/home/ldiamond/dev/maildir/" }
html2text = "0.1.13"
#maildir = "0.4.2"
html2text = "0.2.0"
#html2text = { git = "https://github.com/lewisdiamond/rust-html2text.git"}
mailparse = "0.13.0"
rayon = "1.3.1"
tantivy = "0.12.0"
#mailparse = "0.13.0"
mailparse = { path = "/home/ldiamond/dev/mailparse/" }
rayon = "1.4.0"
tantivy = "0.13.0"
tempdir = "0.3.7"
serde_json = "1.0.57"
serde = { version = "1.0.114", features = ["derive"] }
structopt = "0.3.15"
serde = { version = "1.0.115", features = ["derive"] }
structopt = "0.3.17"
shellexpand = "2.0.0"
log = { version = "0.4.11", features = ["max_level_debug", "release_max_level_warn"] }
pretty_env_logger = "0.4.0"
@ -24,13 +26,15 @@ num_cpus = "1.13.0"
sys-info = "0.7.0"
tui = "0.10.0"
termion = "1.5.5"
chrono = "0.4.13"
chrono = "0.4.15"
sha2 = "0.9.1"
html5ever = "0.25.1"
rocksdb = { path = "../rust-rocksdb/" }
rocksdb = "0.15.0"
jemallocator = "0.3.2"
#maildir = "0.4.2"
select = "0.5.0"
futures = "0.3"
tokio = { version = "0.2", features = ["full"] }
async-trait = "0.1.40"
[dev-dependencies]
rand = "0.7.3"

6
src/bin/readmail_cmd/mod.rs
View File

@ -5,12 +5,12 @@ use structopt::clap::{App, Arg};
fn expand_path(input_str: &str) -> PathBuf {
let expanded = shellexpand::full(input_str)
.expect(format!("Unable to expand {}", input_str).as_str())
.unwrap_or_else(|_| panic!("Unable to expand {}", input_str))
.into_owned();
return PathBuf::from(expanded);
PathBuf::from(expanded)
}
pub fn source() -> Box<BufRead> {
pub fn source() -> Box<dyn BufRead> {
let matches = App::new("Read Mail")
.version("0.0.1")
.author("Lewis Diamond <rms@lewisdiamond.com")

52
src/bin/rms.rs
View File

@ -1,12 +1,13 @@
use log::{error, info, trace};
use rms::cmd::{opts, Command, OutputType};
use rms::message::{Body, Mime};
use rms::stores::{IMessageStore, MessageStoreBuilder, Searchers, Storages};
use rms::terminal;
use std::collections::HashSet;
use std::io::{self, Write};
use std::time::Instant;
fn main() {
#[tokio::main]
pub async fn main() {
pretty_env_logger::init();
let opt = opts();
trace!("Using config file at {:?}", opt.config); //, index.maildir_path);
@ -24,7 +25,7 @@ fn main() {
}
let message_store = MessageStoreBuilder::new()
.storage(Storages::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path))
.debug(debug)
.build();
match message_store {
@ -56,10 +57,9 @@ fn main() {
//message_store.index_mails(full);
}
Command::Search { term, output, num } => {
let now = Instant::now();
let message_store = MessageStoreBuilder::new()
.storage(Storages::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path))
.read_only()
.build();
@ -81,6 +81,20 @@ fn main() {
out.write_all(result.original.as_ref()).unwrap();
}
}
OutputType::Html => {
for m in results {
println!(
"{}",
m.body
.iter()
.filter(|x| x.mime == Mime::Html)
.collect::<Vec<&Body>>()
.first()
.map(|b| b.value.clone())
.unwrap_or_else(|| "No body".to_string())
);
}
}
}
}
Err(e) => error!("{}", e),
@ -102,7 +116,7 @@ fn main() {
Command::Get { id, output } => {
let message_store = MessageStoreBuilder::new()
.storage(Storages::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path))
.read_only()
.build();
@ -133,7 +147,20 @@ fn main() {
.body
.first()
.map(|b| b.value.clone())
.unwrap_or(String::from("No body"))
.unwrap_or_else(|| "No body".to_string())
);
}
OutputType::Html => {
println!(
"{}",
good_msg
.body
.iter()
.filter(|x| x.mime == Mime::Html)
.collect::<Vec<&Body>>()
.first()
.map(|b| b.value.clone())
.unwrap_or_else(|| "No body".to_string())
);
}
},
@ -150,7 +177,7 @@ fn main() {
Command::Latest { num: _num } => {
let message_store = MessageStoreBuilder::new()
.storage(Storages::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path))
.build();
match message_store {
Ok(store) => {
@ -170,13 +197,14 @@ fn main() {
Command::Tag { id, tags } => {
let message_store = MessageStoreBuilder::new()
.storage(Storages::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path.clone()))
.searcher(Searchers::Tantivy(index_dir_path))
.build();
match message_store {
Ok(mut store) => {
match store.tag_message_id(id, tags.into_iter().collect::<HashSet<String>>()) {
Err(e) => error!("{}", e),
Ok(_) => {}
if let Err(e) =
store.tag_message_id(id, tags.into_iter().collect::<HashSet<String>>())
{
error!("{}", e)
}
}
Err(e) => error!("{}", e),

8
src/cmd/mod.rs
View File

@ -9,9 +9,9 @@ pub fn expand_path(input: &OsStr) -> PathBuf {
.to_str()
.expect("Unable to expand the given path. Can't convert input to &str.");
let expanded = shellexpand::full(input_str)
.expect(format!("Unable to expand {}", input_str).as_str())
.unwrap_or_else(|_| panic!("Unable to expand {}", input_str))
.into_owned();
return PathBuf::from(expanded);
PathBuf::from(expanded)
}
#[derive(Debug)]
@ -19,6 +19,7 @@ pub enum OutputType {
Short,
Full,
Raw,
Html,
}
#[derive(Debug)]
@ -38,7 +39,7 @@ impl std::error::Error for OutputTypeError {
"invalid first item to double"
}
fn cause(&self) -> Option<&error::Error> {
fn cause(&self) -> Option<&dyn error::Error> {
// Generic error, underlying cause isn't tracked.
None
}
@ -51,6 +52,7 @@ impl FromStr for OutputType {
"short" => Ok(OutputType::Short),
"full" => Ok(OutputType::Full),
"raw" => Ok(OutputType::Raw),
"html" => Ok(OutputType::Html),
_ => Err(OutputTypeError::UnknownTypeError),
}
}

109
src/message/mod.rs
View File

@ -1,13 +1,13 @@
use crate::readmail;
use crate::readmail::html2text;
use chrono::prelude::*;
use maildir::{MailEntry, ParsedMailEntry};
use maildir::MailEntry;
use mailparse::{dateparse, parse_mail, ParsedMail};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha512};
use std::collections::HashSet;
use std::convert::AsRef;
use std::path::{Path, PathBuf};
use std::fmt;
use std::string::ToString;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -22,20 +22,35 @@ impl Default for Mime {
Mime::PlainText
}
}
impl Mime {
pub fn as_str(&self) -> &str {
match self {
&Mime::PlainText => "text/plain",
&Mime::Html => "text/html",
_ => "Unknown Mime",
impl std::str::FromStr for Mime {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"text/plain" => Ok(Mime::PlainText),
"text/html" => Ok(Mime::Html),
"multipart/alternative" | "multipart/related" => Ok(Mime::Nested),
_ => Ok(Mime::Unknown),
}
}
pub fn from_str(s: &str) -> Mime {
match s {
"text/plain" => Mime::PlainText,
"text/html" => Mime::Html,
"multipart/alternative" | "multipart/related" => Mime::Nested,
_ => Mime::Unknown,
}
impl fmt::Display for Mime {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let msg = match self {
Self::PlainText => "PlainText",
Self::Html => "Html",
Self::Unknown => "Unknown",
Self::Nested => "Nested",
};
write!(f, "Mime:{}", msg)
}
}
impl Mime {
pub fn as_str(&self) -> &str {
match *self {
Mime::PlainText => "text/plain",
Mime::Html => "text/html",
_ => "Unknown Mime",
}
}
}
@ -76,7 +91,7 @@ pub struct ShortMessage {
pub from: String,
pub date: u64,
}
#[allow(dead_code)]
pub struct MessageBuilder {
body: Option<String>,
subject: Option<String>,
@ -87,7 +102,7 @@ pub struct MessageBuilder {
original: Option<Vec<u8>>,
}
pub fn get_id(data: &Vec<u8>) -> String {
pub fn get_id(data: &[u8]) -> String {
format!("{:x}", Sha512::digest(data))
}
@ -103,13 +118,8 @@ impl ToString for Message {
fn to_string(&self) -> String {
let dt = Local.timestamp(self.date as i64, 0);
let dstr = dt.format("%a %b %e %T %Y").to_string();
let tags = if self.tags.len() > 0 {
self.tags
.iter()
.map(|s| s.clone())
.collect::<Vec<String>>()
.join(",")
+ " ||"
let tags = if self.tags.is_empty() {
self.tags.iter().cloned().collect::<Vec<String>>().join(",") + " ||"
} else {
String::from("")
};
@ -122,13 +132,6 @@ impl ToString for Message {
)
}
}
impl AsRef<str> for Message {
fn as_ref(&self) -> &str {
let dt = Local.timestamp(self.date as i64, 0);
let dstr = dt.format("%a %b %e %T %Y").to_string();
"aa" //self.to_string().as_ref()
}
}
pub struct MessageError {
pub message: String,
@ -143,11 +146,8 @@ impl MessageError {
}
impl Message {
pub fn from_parsedmail(
msg: &ParsedMail,
id: String,
original: Vec<u8>,
) -> Result<Self, MessageError> {
pub fn from_parsedmail(msg: &ParsedMail, id: String) -> Result<Self, MessageError> {
let original = Vec::from(msg.data);
let headers = &msg.headers;
let mut subject: String = "".to_string();
let mut from: String = "".to_string();
@ -165,13 +165,16 @@ impl Message {
"Received" | "Date" => {
if date == default_date {
let date_str = h.get_value();
for ts in date_str.rsplit(';') {
date = match dateparse(ts) {
Ok(d) => d,
Err(_) => default_date,
};
break;
}
let date_str = date_str
.rsplit(';')
.collect::<Vec<&str>>()
.first()
.cloned()
.unwrap_or("");
date = match dateparse(date_str) {
Ok(d) => d,
Err(_) => default_date,
};
}
}
_ => {}
@ -194,25 +197,19 @@ impl Message {
let parsed_mail = parse_mail(data.as_slice()).map_err(|_| MessageError {
message: String::from("Unable to parse email data"),
})?;
Self::from_parsedmail(&parsed_mail, id, data.clone())
Self::from_parsedmail(&parsed_mail, id)
}
pub fn from_mailentry(mailentry: MailEntry) -> Result<Self, MessageError> {
let id = mailentry.id();
mailentry.read_data().map_err(|e| MessageError {
message: format!("Failed to parse email id {}", id),
})?;
let data = mailentry.data().ok_or(MessageError {
message: format!("Mail {} could not read data", id),
})?;
pub fn from_mailentry(mut mailentry: MailEntry) -> Result<Self, MessageError> {
let id = mailentry.id().to_owned();
match mailentry.parsed() {
Ok(parsed) => Self::from_parsedmail(&parsed, String::from(id), data.clone()),
Err(e) => Err(MessageError {
Ok(parsed) => Self::from_parsedmail(&parsed, id),
Err(_) => Err(MessageError {
message: format!("Failed to parse email id {}", id),
}),
}
}
pub fn get_body(&self) -> &Body {
self.body.iter().next().unwrap()
self.body.get(0).unwrap()
}
pub fn to_long_string(&self) -> String {
format!(
@ -234,7 +231,7 @@ impl Message {
)
}
}
#[allow(dead_code)]
impl MessageBuilder {
fn body(mut self, body: String) -> Self {
self.body = Some(body);
@ -257,7 +254,7 @@ impl MessageBuilder {
self
}
fn recipients(mut self, recipients: String) -> Self {
self.recipients = Some(recipients.split(",").map(|s| String::from(s)).collect());
self.recipients = Some(recipients.split(',').map(String::from).collect());
self
}
fn original(mut self, original: Vec<u8>) -> Self {

31
src/readmail/mod.rs
View File

@ -1,5 +1,5 @@
extern crate select;
use crate::message::{get_id, Body, Message, Mime};
use crate::message::{Body, Mime};
use log::debug;
use mailparse::*;
use select::document::Document;
@ -23,24 +23,22 @@ pub fn extract_body(msg: &ParsedMail, prefer_html: bool) -> Vec<Body> {
} else {
Mime::PlainText
};
let text = msg
.get_body()
.unwrap_or(msg.get_body_raw().map_or(String::from(""), |x| {
String::from_utf8(x).unwrap_or(String::from(""))
}));
let mime = Mime::from_str(&msg.ctype.mimetype);
let text = msg.get_body().unwrap_or_else(|_| {
msg.get_body_raw().map_or(String::from(""), |x| {
String::from_utf8(x).unwrap_or_else(|_| String::new())
})
});
let mime = msg.ctype.mimetype.parse::<Mime>().unwrap();
let raw_body = Some(Body::new(mime, text));
let mut bodies = msg
.subparts
.iter()
.map(|mut s| {
let mime = Mime::from_str(&s.ctype.mimetype);
.map(|s| {
let mime = s.ctype.mimetype.parse::<Mime>().unwrap();
match mime {
Mime::PlainText | Mime::Html => {
s.get_body().ok().map(|b| Body::new(mime, String::from(b)))
}
Mime::Nested => extract_body(&mut s, prefer_html).into_iter().next(),
Mime::PlainText | Mime::Html => s.get_body().ok().map(|b| Body::new(mime, b)),
Mime::Nested => extract_body(&s, prefer_html).into_iter().next(),
Mime::Unknown => {
debug!("unknown mime {}", mime.as_str());
None
@ -53,7 +51,7 @@ pub fn extract_body(msg: &ParsedMail, prefer_html: bool) -> Vec<Body> {
bodies.push(raw_body.expect("COULD NOT UNWRAP RAW_BODY"));
}
bodies.sort_unstable_by(|x, y| cmp_body(x, y, &prefered_mime));
if bodies.len() == 0 {
if bodies.is_empty() {
println!(
"No body for message: {}",
msg.headers
@ -70,7 +68,8 @@ pub fn html2text(text: &str) -> String {
let document = Document::from(text);
let text_nodes = document
.find(Text)
.map(|x| x.text())
.map(|x| String::from(x.text().trim()))
.filter(|x| x.len() > 1)
.collect::<Vec<String>>();
return text_nodes.join("\n\n");
text_nodes.join("\n\n")
}

42
src/stores/_impl/rocksdb.rs
View File

@ -1,28 +1,21 @@
use crate::message::{Message, ShortMessage};
use crate::message::Message;
use crate::stores::{IMessageStorage, MessageStoreError};
use chrono::{DateTime, Utc};
use rocksdb::{DBCompactionStyle, DBCompressionType};
use rocksdb::{DBVector, Options, DB};
use serde_json::Result as SResult;
use std::collections::HashSet;
use rocksdb::{Options, DB};
use std::path::Path;
use std::string::ToString;
type RocksDBMessage = Message;
impl RocksDBMessage {
fn from_rocksdb(msg: DBVector) -> Result<RocksDBMessage, MessageStoreError> {
let msg_r = msg
.to_utf8()
.ok_or(Err(MessageStoreError::CouldNotGetMessage(
"Message is malformed in some way".to_string(),
)));
fn from_rocksdb(msg: Vec<u8>) -> Result<RocksDBMessage, MessageStoreError> {
let msg = String::from_utf8(msg).map_err(|_| {
MessageStoreError::CouldNotGetMessage("Message is malformed in some way".to_string())
})?;
match msg_r {
Ok(msg) => serde_json::from_str(msg).map_err(|e| {
MessageStoreError::CouldNotGetMessage("Unable to parse the value".to_string())
}),
Err(e) => e,
}
serde_json::from_str(&msg).map_err(|_| {
MessageStoreError::CouldNotGetMessage("Unable to parse the value".to_string())
})
}
fn to_rocksdb(&self) -> Result<(String, Vec<u8>), MessageStoreError> {
let id = self.id.clone();
@ -71,29 +64,30 @@ impl IMessageStorage for RocksDBStore {
))),
}
}
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
fn update_message(&mut self, _msg: Message) -> Result<Message, MessageStoreError> {
unimplemented!()
}
fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
fn delete_message(&mut self, _msg: Message) -> Result<(), MessageStoreError> {
unimplemented!()
}
fn get_messages_page(
&self,
start: usize,
_start: usize,
num: usize,
) -> Result<Vec<Message>, MessageStoreError> {
Ok(self.latest(num))
}
fn get_by_date(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
_start: DateTime<Utc>,
_end: DateTime<Utc>,
) -> Result<Vec<Message>, MessageStoreError> {
unimplemented!()
}
}
impl RocksDBStore {
#[allow(dead_code)]
pub fn new<P: AsRef<Path>>(path: P) -> Self {
let mut opts = Options::default();
opts.increase_parallelism(16);
@ -156,8 +150,8 @@ mod test {
fn drop(&mut self) {
let opts = Options::default();
let path = self.path.as_ref().unwrap();
DB::destroy(&opts, path);
std::fs::remove_dir_all(path);
DB::destroy(&opts, path).unwrap();
std::fs::remove_dir_all(path).unwrap();
}
}
#[test]
@ -191,6 +185,6 @@ mod test {
let store = &StoreInit::new().store;
store.db.put(b"key", b"value2").unwrap();
let get = store.db.get(b"key").ok().unwrap().unwrap();
assert_eq!("value2", get.to_utf8().unwrap());
assert_eq!("value2", String::from_utf8(get).unwrap());
}
}

134
src/stores/_impl/tantivy.rs
View File

@ -1,30 +1,26 @@
use crate::stores::{IMessageSearcher, IMessageStorage, MessageStoreError};
use chrono::{DateTime, Utc};
use crate::message::{Message, MessageError};
use crate::stores::{IMessageSearcher, MessageStoreError};
use log::info;
use std::cmp;
use std::collections::HashSet;
use std::fs;
use std::panic;
use std::time::Instant;
use crate::message::{Body, Message, Mime};
use std::path::PathBuf;
use std::string::ToString;
use tantivy;
use tantivy::collector::{Count, TopDocs};
use tantivy::collector::TopDocs;
use tantivy::directory::MmapDirectory;
use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, RangeQuery, TermQuery};
use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, TermQuery};
use tantivy::schema::*;
const BYTES_IN_MB: usize = 1024 * 1024;
pub type TantivyMessage = Message;
pub trait TantivyFrom<T> {
fn from_tantivy(doc: Document, schema: &EmailSchema) -> T;
fn from_tantivy(doc: Document, schema: &EmailSchema) -> Result<T, MessageError>;
}
impl TantivyFrom<TantivyMessage> for TantivyMessage {
fn from_tantivy(doc: Document, schema: &EmailSchema) -> TantivyMessage {
fn from_tantivy(doc: Document, schema: &EmailSchema) -> Result<TantivyMessage, MessageError> {
let original: Result<Vec<u8>, _> = match doc
.get_first(schema.original)
.expect("Unable to get original message")
@ -33,55 +29,15 @@ impl TantivyFrom<TantivyMessage> for TantivyMessage {
_ => Err("Missing original email from the index"),
};
let tags: HashSet<String> = doc
let _tags: HashSet<String> = doc
.get_all(schema.tag)
.into_iter()
.filter_map(|s| s.text())
.map(|s| String::from(s))
.map(String::from)
.collect();
TantivyMessage {
id: doc
.get_first(schema.id)
.expect("Message without an id")
.text()
.expect("Message ID is always a string")
.to_string(),
from: String::from(
doc.get_first(schema.from)
.expect("Message without from")
.text()
.expect("Message with non-text from"),
),
subject: String::from(
doc.get_first(schema.subject)
.expect("Message without subject")
.text()
.expect("Message with non-text subject"),
),
date: doc
.get_first(schema.date)
.map_or(0, |v: &tantivy::schema::Value| v.u64_value()),
recipients: doc
.get_first(schema.recipients)
.unwrap_or(&tantivy::schema::Value::Str(String::from("a")))
.text()
.expect("Message with non-text recipients")
.split(",")
.map(|s| String::from(s))
.collect(),
body: vec![Body {
mime: Mime::PlainText,
value: String::from(
doc.get_first(schema.body)
.expect("Message without body")
.text()
.expect("Message with non-text body"),
),
}],
original: original.expect("Original was missing from the index"),
tags,
}
TantivyMessage::from_data(
original.map_err(|_| MessageError::from("Could not read original from index"))?,
)
}
}
@ -91,6 +47,7 @@ pub struct EmailSchema {
body: Field,
from: Field,
recipients: Field,
#[allow(dead_code)]
thread: Field,
id: Field,
date: Field,
@ -157,7 +114,7 @@ impl IMessageSearcher for TantivyStore {
match writer {
Some(writer) => match writer.commit() {
Ok(_) => Ok(()),
Err(e) => Err(MessageStoreError::CouldNotAddMessage(
Err(_) => Err(MessageStoreError::CouldNotAddMessage(
"Failed to commit to index".to_string(),
)),
},
@ -181,10 +138,10 @@ impl IMessageSearcher for TantivyStore {
) -> Result<Vec<TantivyMessage>, MessageStoreError> {
Ok(self.search(query.as_str(), num))
}
fn delete_message(&mut self, msg: &Message) -> Result<(), MessageStoreError> {
fn delete_message(&mut self, _msg: &Message) -> Result<(), MessageStoreError> {
Ok(())
}
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
fn update_message(&mut self, _msg: Message) -> Result<Message, MessageStoreError> {
unimplemented!();
}
@ -207,7 +164,7 @@ impl TantivyStore {
pub fn new(path: PathBuf) -> Self {
Self::_new(path, false)
}
fn _new(path: PathBuf, ro: bool) -> Self {
fn _new(path: PathBuf, _ro: bool) -> Self {
let email = EmailSchema::default();
let index = TantivyStore::open_or_create_index(path, email.schema.clone());
let reader = index
@ -222,22 +179,21 @@ impl TantivyStore {
mem_per_thread: None,
}
}
pub fn new_ro(path: PathBuf) -> Self {
pub fn _new_ro(path: PathBuf) -> Self {
Self::_new(path, true)
}
fn open_or_create(path: PathBuf, schema: Schema) -> tantivy::Index {
tantivy::Index::open_or_create(MmapDirectory::open(path).unwrap(), schema.clone()).unwrap()
tantivy::Index::open_or_create(MmapDirectory::open(path).unwrap(), schema).unwrap()
}
fn open_or_create_index(path: PathBuf, schema: Schema) -> tantivy::Index {
fs::create_dir_all(path.as_path()).expect(
format!(
fs::create_dir_all(path.as_path()).unwrap_or_else(|_| {
panic!(
"Unable to create or access the given index directory {}",
path.to_str().unwrap()
)
.as_str(),
);
});
TantivyStore::open_or_create(path, schema)
}
@ -262,7 +218,7 @@ impl TantivyStore {
.iter()
.for_each(|t| document.add_text(email.tag, t.as_str()));
indexer.add_document(document);
Ok(msg.id.clone())
Ok(msg.id)
}
None => Err(MessageStoreError::CouldNotAddMessage(
"No indexer was allocated".to_string(),
@ -274,7 +230,7 @@ impl TantivyStore {
match writer {
Some(indexer) => {
let term = Term::from_field_text(self.email.id, msg.id.as_ref());
indexer.delete_term(term.clone());
indexer.delete_term(term);
Ok(())
}
None => Err(MessageStoreError::CouldNotModifyMessage(
@ -282,11 +238,15 @@ impl TantivyStore {
)),
}
}
pub fn _tag_doc(&self, doc: Document, tags: Vec<String>) -> Result<(), tantivy::TantivyError> {
pub fn _tag_doc(&self, doc: Document, _tags: Vec<String>) -> Result<(), tantivy::TantivyError> {
let mut writer = self.get_index_writer(1).ok().unwrap();
let id = TantivyMessage::from_tantivy(doc, &self.email).id;
let id = TantivyMessage::from_tantivy(doc, &self.email)
.map_err(|_| {
tantivy::TantivyError::SystemError(String::from("Can't read message from index"))
})?
.id;
let term = Term::from_field_text(self.email.id, id.as_ref());
writer.delete_term(term.clone());
writer.delete_term(term);
writer.commit()?;
self.reader.reload()
}
@ -299,10 +259,7 @@ impl TantivyStore {
Some(threads) => threads,
None => cmp::min(
(num_cpus::get() as f32 / 1.5).floor() as usize,
cmp::max(
1,
(0.0818598 * (num_emails as f32).powf(0.31154938)) as usize,
),
cmp::max(1, (0.0818598 * (num_emails as f32).powf(0.311549)) as usize),
),
};
let mem_per_thread = match self.mem_per_thread {
@ -313,7 +270,7 @@ impl TantivyStore {
cmp::min(
mem_info.avail as usize * 1024 / (num_cpu + 1),
cmp::max(
(0.41268337 * (num_emails as f32).powf(0.67270258)) as usize
(0.41268337 * (num_emails as f32).powf(0.672702)) as usize
* BYTES_IN_MB,
200 * BYTES_IN_MB,
),
@ -335,7 +292,7 @@ impl TantivyStore {
.writer_with_num_threads(num_cpu, mem_per_thread * num_cpu)
{
Ok(index_writer) => Ok(index_writer),
Err(e) => Err(MessageStoreError::CouldNotAddMessage(
Err(_) => Err(MessageStoreError::CouldNotAddMessage(
"Impossible to create the indexer".to_string(),
)),
}
@ -353,14 +310,18 @@ impl TantivyStore {
&AllQuery,
&TopDocs::with_limit(num + skip).order_by_u64_field(self.email.date),
)
.map_err(|e| MessageStoreError::CouldNotGetMessages(vec![]))?;
.map_err(|_| MessageStoreError::CouldNotGetMessages(vec![]))?;
let mut ret = vec![];
let page = docs
.drain(skip..)
.collect::<Vec<(u64, tantivy::DocAddress)>>();
for doc in page {
let retrieved_doc = searcher.doc(doc.1).unwrap();
ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
ret.push(
TantivyMessage::from_tantivy(retrieved_doc, &self.email).map_err(|_| {
MessageStoreError::CouldNotGetMessage("Message is corrupt".to_string())
})?,
);
}
Ok(ret)
}
@ -369,7 +330,7 @@ impl TantivyStore {
// Is this needed? self.reader.load_searchers()?;
let searcher = self.reader.searcher();
let termq = TermQuery::new(
Term::from_field_text(self.email.id, id.as_ref()),
Term::from_field_text(self.email.id, id),
IndexRecordOption::Basic,
);
let addr = searcher.search(&termq, &TopDocs::with_limit(1));
@ -387,8 +348,8 @@ impl TantivyStore {
pub fn _get_message(&self, id: &str) -> Option<TantivyMessage> {
let doc = self.get_doc(id);
match doc {
Ok(doc) => Some(TantivyMessage::from_tantivy(doc, &self.email)),
Err(_) => None,
Ok(doc) => TantivyMessage::from_tantivy(doc, &self.email).ok(),
_ => None,
}
}
pub fn search(&self, text: &str, num: usize) -> Vec<TantivyMessage> {
@ -401,11 +362,14 @@ impl TantivyStore {
let mut ret = vec![];
for doc in top_docs {
let retrieved_doc = searcher.doc(doc.1).unwrap();
ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
if let Ok(d) = TantivyMessage::from_tantivy(retrieved_doc, &self.email) {
ret.push(d);
}
}
ret
}
#[allow(dead_code)]
pub fn fuzzy(&self, text: &str, num: usize) -> Vec<TantivyMessage> {
let mut terms = text.split(' ').collect::<Vec<&str>>();
terms.insert(0, text);
@ -413,7 +377,7 @@ impl TantivyStore {
let mut ret = self.search(text, num);
for n in 1..2 {
if ret.len() < num {
let mut queries: Vec<(Occur, Box<Query>)> = vec![];
let mut queries: Vec<(Occur, Box<dyn Query>)> = vec![];
for (_, t) in terms.iter().enumerate() {
let term = Term::from_field_text(self.email.subject, t);
let term_body = Term::from_field_text(self.email.body, t);
@ -427,7 +391,9 @@ impl TantivyStore {
let top_docs = searcher.search(&bquery, &top_docs_by_date).unwrap();
for doc in top_docs {
let retrieved_doc = searcher.doc(doc.1).unwrap();
ret.push(TantivyMessage::from_tantivy(retrieved_doc, &self.email));
if let Ok(d) = TantivyMessage::from_tantivy(retrieved_doc, &self.email) {
ret.push(d);
}
}
}
}

24
src/stores/message_store.rs
View File

@ -1,4 +1,4 @@
use crate::message::{Message, ShortMessage};
use crate::message::Message;
use crate::stores::{IMessageSearcher, IMessageStorage, IMessageStore, MessageStoreError};
use chrono::{DateTime, Utc};
use log::error;
@ -9,7 +9,7 @@ use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::mpsc;
use std::thread;
use std::time::{Duration, Instant};
use std::time::Duration;
pub struct MessageStore {
pub searcher: Box<dyn IMessageSearcher>,
@ -37,21 +37,21 @@ impl IMessageStore for MessageStore {
}
fn tag_message_id(
&mut self,
id: String,
tags: HashSet<String>,
_id: String,
_tags: HashSet<String>,
) -> Result<usize, MessageStoreError> {
unimplemented!();
}
fn tag_message(
&mut self,
msg: Message,
tags: HashSet<String>,
_msg: Message,
_tags: HashSet<String>,
) -> Result<usize, MessageStoreError> {
unimplemented!();
}
fn update_message(&mut self, msg: Message) -> Result<Message, MessageStoreError> {
fn update_message(&mut self, _msg: Message) -> Result<Message, MessageStoreError> {
unimplemented!();
}
fn get_messages_page(
@ -67,12 +67,12 @@ impl IMessageStore for MessageStore {
}
fn search_by_date(
&self,
start: DateTime<Utc>,
end: DateTime<Utc>,
_start: DateTime<Utc>,
_end: DateTime<Utc>,
) -> Result<Vec<Message>, MessageStoreError> {
unimplemented!();
}
fn delete_message(&mut self, msg: Message) -> Result<(), MessageStoreError> {
fn delete_message(&mut self, _msg: Message) -> Result<(), MessageStoreError> {
unimplemented!();
}
}
@ -110,7 +110,7 @@ impl MessageStore {
}
fn init_progress(&mut self, num: usize) -> thread::JoinHandle<()> {
let mut mb = MultiBar::new();
let mb = MultiBar::new();
mb.println(&format!("Indexing {} emails", num));
let mut index_bar = mb.create_bar(num as u64);
if num < 10_000_000 {
@ -190,7 +190,7 @@ impl MessageStore {
self.do_index_mails(maildir, full)?;
Ok(1)
}
Err(e) => Err(MessageStoreError::CouldNotOpenMaildir(
Err(_) => Err(MessageStoreError::CouldNotOpenMaildir(
"Failed to read maildir".to_string(),
)),
}

10
src/stores/mod.rs
View File

@ -3,12 +3,10 @@ use chrono::{DateTime, Utc};
use std::collections::HashSet;
use std::path::PathBuf;
mod _impl;
use _impl::rocksdb;
use _impl::tantivy;
use std::fmt;
pub mod message_store;
use message_store::MessageStore;
use std::time::Instant;
pub enum Searchers {
Tantivy(PathBuf),
@ -40,7 +38,11 @@ impl fmt::Display for MessageStoreBuilderError {
}
}
}
impl Default for MessageStoreBuilder {
fn default() -> Self {
MessageStoreBuilder::new()
}
}
impl MessageStoreBuilder {
pub fn new() -> MessageStoreBuilder {
MessageStoreBuilder {
@ -109,7 +111,7 @@ impl MessageStoreBuilder {
Searchers::Tantivy(path) => {
let mut p = path.clone();
p.push("searcher");
Ok(tantivy::TantivyStore::new(std::path::PathBuf::from(p)))
Ok(tantivy::TantivyStore::new(p))
}
},
};

6
src/terminal/input/list.rs
View File

@ -59,11 +59,11 @@ impl Runnable for ListRunner {
}
}
pub fn handler() -> Box<InputHandler> {
Box::new(InputHandler {
pub fn handler() -> InputHandler {
InputHandler {
name: String::from("List"),
pre: true,
f: Box::new(ListRunner {}),
children: vec![],
})
}
}

32
src/terminal/input/mod.rs
View File

@ -8,8 +8,8 @@ mod search;
pub struct InputHandler {
pub name: String,
pre: bool,
f: Box<Runnable>,
children: Vec<Box<InputHandler>>,
f: Box<dyn Runnable>,
children: Vec<InputHandler>,
}
#[derive(Debug)]
@ -26,15 +26,11 @@ impl Runnable for CtrlCRunner {
Event::Input(key) => match key {
Key::Ctrl('c') => {
store.exit = true;
return true;
}
_ => {
return false;
true
}
_ => false,
},
_ => {
return false;
}
_ => false,
}
}
}
@ -46,15 +42,11 @@ impl Runnable for QExitRunner {
Event::Input(key) => match key {
Key::Char('q') => {
store.exit = true;
return true;
}
_ => {
return false;
true
}
_ => false,
},
_ => {
return false;
}
_ => false,
}
}
}
@ -76,7 +68,7 @@ pub trait Runnable {
}
impl InputHandler {
pub fn new(name: String, children: Vec<Box<InputHandler>>) -> InputHandler {
pub fn new(name: String, children: Vec<InputHandler>) -> InputHandler {
Self {
name,
f: Box::new(CtrlCRunner {}),
@ -84,13 +76,13 @@ impl InputHandler {
children,
}
}
pub fn new_single(name: String, f: Box<Runnable>, pre: bool) -> Box<InputHandler> {
Box::new(Self {
pub fn new_single(name: String, f: Box<dyn Runnable>, pre: bool) -> InputHandler {
Self {
name,
f,
pre,
children: vec![],
})
}
}
fn input(&self, e: &Event<Key>, store: &mut Store) -> bool {
if self.pre {

6
src/terminal/input/reader.rs
View File

@ -48,11 +48,11 @@ impl Runnable for ReaderRunner {
}
}
pub fn handler() -> Box<InputHandler> {
Box::new(InputHandler {
pub fn handler() -> InputHandler {
InputHandler {
name: String::from("Reader"),
pre: true,
f: Box::new(ReaderRunner {}),
children: vec![],
})
}
}

22
src/terminal/input/search.rs
View File

@ -31,36 +31,30 @@ impl Runnable for SearchRunner {
store.search_store.set_search("".to_string());
true
}
_ => {
return false;
}
_ => false,
},
_ => {
return false;
}
_ => false,
}
} else {
match e {
Event::Input(key) => match key {
Key::Char('/') => {
store.search_store.enable_search();
return true;
}
_ => {
return false;
true
}
_ => false,
},
_ => return false,
_ => false,
}
}
}
}
pub fn handler() -> Box<InputHandler> {
Box::new(InputHandler {
pub fn handler() -> InputHandler {
InputHandler {
name: String::from("Search"),
pre: true,
f: Box::new(SearchRunner {}),
children: vec![],
})
}
}

8
src/terminal/store/list.rs
View File

@ -7,11 +7,11 @@ pub struct ListStore<'a> {
pub page_size: usize,
pub curr_idx: usize,
pub fetched_first: bool,
pub message_store: &'a IMessageStore,
pub message_store: &'a dyn IMessageStore,
}
impl<'a> ListStore<'a> {
pub fn new(msg_store: &'a IMessageStore) -> ListStore<'a> {
pub fn new(msg_store: &'a dyn IMessageStore) -> ListStore<'a> {
ListStore {
messages: vec![],
selected: 0,
@ -40,7 +40,7 @@ impl<'a> ListStore<'a> {
}
pub fn prev_page(&mut self) -> &Self {
self.set_selected(-1 * self.page_size as i32);
self.set_selected(-(self.page_size as i32));
self
}
pub fn set_selected(&mut self, offset: i32) -> &Self {
@ -66,7 +66,7 @@ impl<'a> ListStore<'a> {
.get_messages_page(self.curr_idx, page_size);
match messages {
Ok(messages) => self.messages = messages,
Err(e) => self.messages = vec![], // TODO Handle error
Err(_) => self.messages = vec![], // TODO Handle error
}
}
}

2
src/terminal/store/mod.rs
View File

@ -16,7 +16,7 @@ pub struct Store<'a> {
pub tags_store: TagsStore<'a>,
}
impl<'a> Store<'a> {
pub fn new(message_store: &'a IMessageStore) -> Store {
pub fn new(message_store: &'a dyn IMessageStore) -> Store {
Store {
exit: false,
search_store: SearchStore::new(message_store),

4
src/terminal/store/reader.rs
View File

@ -5,11 +5,11 @@ use std::cmp::max;
pub struct ReaderStore<'a> {
pub message: Option<Message>,
pub scroll: u16,
pub storage: &'a IMessageStore,
pub storage: &'a dyn IMessageStore,
}
impl<'a> ReaderStore<'a> {
pub fn new(storage: &'a IMessageStore) -> ReaderStore<'a> {
pub fn new(storage: &'a dyn IMessageStore) -> ReaderStore<'a> {
ReaderStore {
message: None,
scroll: 0,

4
src/terminal/store/search.rs
View File

@ -4,11 +4,11 @@ use crate::stores::IMessageStore;
pub struct SearchStore<'a> {
pub search_term: String,
pub searching: bool,
pub searcher: &'a IMessageStore,
pub searcher: &'a dyn IMessageStore,
pub results: Vec<Message>,
}
impl<'a> SearchStore<'a> {
pub fn new(msg_store: &'a IMessageStore) -> SearchStore {
pub fn new(msg_store: &'a dyn IMessageStore) -> SearchStore {
SearchStore {
search_term: String::from(""),
searching: false,

4
src/terminal/store/tags.rs
View File

@ -2,11 +2,11 @@ use crate::message::Message;
use crate::stores::IMessageStore;
pub struct TagsStore<'a> {
pub message_store: &'a IMessageStore,
pub message_store: &'a dyn IMessageStore,
pub message: Option<Message>,
}
impl<'a> TagsStore<'a> {
pub fn new(msg_store: &'a IMessageStore) -> TagsStore<'a> {
pub fn new(msg_store: &'a dyn IMessageStore) -> TagsStore<'a> {
TagsStore {
message: None,
message_store: msg_store,

23
src/terminal/views/email_read.rs
View File

@ -1,13 +1,12 @@
use crate::message::Message;
use tui::backend::Backend;
use tui::layout::Rect;
use tui::style::{Modifier, Style};
use tui::widgets::{Block, Borders, Paragraph, Text, Widget};
use tui::text::Text;
use tui::widgets::{Block, Borders, Paragraph, Wrap};
use tui::Frame;
pub fn draw<B: Backend>(f: &mut Frame<B>, message: &Message, scroll: u16) {
let text = message.to_long_string();
let text = [Text::raw(text)];
let f_r = f.size();
let rect = Rect {
x: f_r.x + f_r.width / 2 - 40,
@ -16,16 +15,14 @@ pub fn draw<B: Backend>(f: &mut Frame<B>, message: &Message, scroll: u16) {
height: f_r.height,
};
let block = Block::default()
.borders(Borders::ALL)
.title_style(Style::default().modifier(Modifier::BOLD));
Paragraph::new(text.iter())
let block = Block::default().borders(Borders::ALL);
let p = Paragraph::new(Text::from(text.as_str()))
.block(block.clone())
.wrap(true)
.scroll(scroll)
.render(f, rect);
.wrap(Wrap { trim: true })
.scroll((scroll, 0));
f.render_widget(p, rect);
Paragraph::new([Text::raw(format!("scroll {}", scroll))].iter())
.wrap(true)
.render(f, Rect::new(0, 0, 100, 2));
let p2_str = format!("scroll {}", scroll);
let p2 = Paragraph::new(Text::from(p2_str.as_str())).wrap(Wrap { trim: true });
f.render_widget(p2, Rect::new(0, 0, 100, 2));
}

15
src/terminal/views/mod.rs
View File

@ -2,7 +2,8 @@ use crate::terminal::store::Store;
use std::io;
use tui::backend::Backend;
use tui::layout::{Constraint, Direction, Layout};
use tui::widgets::{Block, Borders, Paragraph, Text, Widget};
use tui::text::Text;
use tui::widgets::{Block, Borders, Paragraph, Wrap};
use tui::Terminal;
pub mod email_read;
pub mod search_results;
@ -25,16 +26,14 @@ pub fn draw<B: Backend>(terminal: &mut Terminal<B>, store: &Store) -> Result<(),
.constraints([Constraint::Length(40), Constraint::Min(100)].as_ref())
.split(main[0]);
if store.search_store.searching {
Paragraph::new([Text::raw(store.search_store.search_term.as_str())].iter())
let s = Paragraph::new(Text::from(store.search_store.search_term.as_str()))
.block(Block::default().title("Search").borders(Borders::ALL))
.wrap(true)
.render(&mut f, main[1]);
.wrap(Wrap { trim: true });
f.render_widget(s, main[1]);
}
Block::default()
.title("Tags")
.borders(Borders::ALL)
.render(&mut f, chunks[0]);
let t = Block::default().title("Tags").borders(Borders::ALL);
f.render_widget(t, chunks[0]);
search_results::draw(&mut f, chunks[1], &store);
}
})

27
src/terminal/views/search_results.rs
View File

@ -2,7 +2,8 @@ use crate::terminal::store::Store;
use tui::backend::Backend;
use tui::layout::Rect;
use tui::style::{Color, Modifier, Style};
use tui::widgets::{Block, Borders, SelectableList, Widget};
use tui::text::Span;
use tui::widgets::{Block, Borders, List, ListItem, ListState};
use tui::Frame;
pub fn draw<B: Backend>(f: &mut Frame<B>, area: Rect, store: &Store) {
@ -12,17 +13,19 @@ pub fn draw<B: Backend>(f: &mut Frame<B>, area: Rect, store: &Store) {
} else {
&store.search_store.results
};
SelectableList::default()
let mut state = ListState::default();
let items: Vec<ListItem> = display
.iter()
.map(|s| {
let s = s.to_string();
ListItem::new(Span::raw(s))
})
.collect::<Vec<ListItem>>();
let list = List::new(items)
.block(Block::default().borders(Borders::ALL).title("List"))
.items(
&display
.iter()
.map(|s| s.to_string())
.collect::<Vec<String>>(),
)
.select(Some(store.list_store.selected))
.style(style)
.highlight_style(style.fg(Color::LightGreen).modifier(Modifier::BOLD))
.highlight_symbol(">")
.render(f, area);
.highlight_style(style.fg(Color::LightGreen).add_modifier(Modifier::BOLD))
.highlight_symbol(">");
state.select(Some(store.list_store.selected));
f.render_stateful_widget(list, area, &mut state);
}