Partially working migration to async/await

Cargo.toml

@@ -8,33 +8,33 @@ edition = "2018"
 #maildir = { git = "https://github.com/lewisdiamond/maildir.git" }
 maildir = { path = "/home/ldiamond/dev/maildir/" }
 #maildir = "0.4.2"
-html2text = "0.2.0"
+html2text = "0.2.1"
 #html2text = { git = "https://github.com/lewisdiamond/rust-html2text.git"}
 #mailparse = "0.13.0"
 mailparse = { path = "/home/ldiamond/dev/mailparse/" }
-rayon = "1.4.0"
+rayon = "1.5.0"
-tantivy = "0.13.0"
+tantivy = "0.13.2"
 tempdir = "0.3.7"
-serde_json = "1.0.57"
+serde_json = "1.0.61"
-serde = { version = "1.0.115", features = ["derive"] }
+serde = { version = "1.0.118", features = ["derive"] }
-structopt = "0.3.17"
+structopt = "0.3.21"
-shellexpand = "2.0.0"
+shellexpand = "2.1.0"
 log = { version = "0.4.11", features = ["max_level_debug", "release_max_level_warn"] } 
 pretty_env_logger = "0.4.0"
 pbr = "1.0.3"
 num_cpus = "1.13.0"
 sys-info = "0.7.0"
-tui = "0.10.0"
+tui = "0.13.0"
 termion = "1.5.5"
-chrono = "0.4.15"
+chrono = "0.4.19"
-sha2 = "0.9.1"
+sha2 = "0.9.2"
 html5ever = "0.25.1"
 rocksdb = "0.15.0"
 jemallocator = "0.3.2"
 select = "0.5.0"
-futures = "0.3"
+futures = "0.3.8"
-tokio = { version = "0.2", features = ["full"] }
+tokio = { version = "1.0.1", features = ["full"] }
-async-trait = "0.1.40"
+async-trait = "0.1.42"
 
 [dev-dependencies]
-rand = "0.7.3"
+rand = "0.8.0"

src/bin/readmail.rs

@@ -18,15 +18,13 @@ fn main() {
                         println!("From: {}", message.from);
                         println!("To: {}", message.recipients.join(", "));
                         println!("Subject: {}", message.subject);
-                        for b in message.body {
+                        let body = message.get_body(None);
-                            println!("Body Mime: {}", b.mime.as_str());
+                        match body.mime {
-                            match b.mime {
+                            Mime::PlainText => println!("\n\n{}", body.value),
-                                Mime::PlainText => println!("\n\n{}", b.value),
+                            Mime::Html => println!("\n\n{}", html2text(&body.value)),
-                                Mime::Html => println!("\n\n{}", html2text(&b.value)),
                             _ => println!("Unknown mime type"),
                         }
                     }
-                    }
                     Err(_e) => error!("Failed to make sense of the message"),
                 }
             }

src/bin/rms.rs

@@ -1,5 +1,6 @@
 use log::{error, info, trace};
-use rms::cmd::{opts, Command, OutputType};
+use rms::cmd::{opts, Command};
+use rms::readmail::display::{OutputType, DisplayAs};
 use rms::message::{Body, Mime};
 use rms::stores::{IMessageStore, MessageStoreBuilder, Searchers, Storages};
 use rms::terminal;
@@ -25,27 +26,27 @@ pub async fn main() {
             }
             let message_store = MessageStoreBuilder::new()
                 .storage(Storages::Tantivy(index_dir_path.clone()))
-                .searcher(Searchers::Tantivy(index_dir_path))
+                .searcher(Searchers::Tantivy(index_dir_path.clone()))
                 .debug(debug)
                 .build();
             match message_store {
                 Ok(mut store) => {
-                    maildir_path.into_iter().for_each(|m| {
+                    for m in maildir_path {
                         println!("Adding maildir at {}", m.to_str().unwrap());
-                        match store.add_maildir(m.clone(), full) {
+                        match store.add_maildir(m.clone(), full).await {
                             Err(e) => error!(
                                 "Failed to add mails from {}, detauls: {}",
                                 m.to_str().unwrap(),
                                 e
                             ),
                             Ok(_) => println!("Successfully added {}", m.to_str().unwrap()),
+                        };
                     }
-                    });
                 }
                 Err(e) => {
                     error!("{}", e);
                 }
-            }
+            };
             //maildir_path[0].clone(), index_dir_path);
             //if let Some(threads) = threads {
             //    indexer_builder.threads(threads);
@@ -56,7 +57,7 @@ pub async fn main() {
             //let mut indexer = indexer_builder.build();
             //message_store.index_mails(full);
         }
-        Command::Search { term, output, num } => {
+        Command::Search { term, output, num, advanced } => {
             let message_store = MessageStoreBuilder::new()
                 .storage(Storages::Tantivy(index_dir_path.clone()))
                 .searcher(Searchers::Tantivy(index_dir_path))
@@ -66,36 +67,39 @@ pub async fn main() {
             match message_store {
                 Ok(store) => {
                     let results = store.search_fuzzy(term, num).ok().unwrap();
-                    match output {
-                        OutputType::Short => {
                         for r in results {
-                                println!("{:?} | {}", r.id, r.subject);
+                            print!("{}", r.display(&output));
-                            }
-                        }
-                        OutputType::Full => {
-                            println!("{:?}", results);
-                        }
-                        OutputType::Raw => {
-                            let mut out = io::stdout();
-                            for result in results {
-                                out.write_all(result.original.as_ref()).unwrap();
-                            }
-                        }
-                        OutputType::Html => {
-                            for m in results {
-                                println!(
-                                    "{}",
-                                    m.body
-                                        .iter()
-                                        .filter(|x| x.mime == Mime::Html)
-                                        .collect::<Vec<&Body>>()
-                                        .first()
-                                        .map(|b| b.value.clone())
-                                        .unwrap_or_else(|| "No body".to_string())
-                                );
-                            }
-                        }
                         }
+                    //match output {
+ //                       OutputType::Short => {
+ //                           for r in results {
+ //                               println!("{:?} | {}", r.id, r.subject);
+ //                           }
+ //                       }
+ //                       OutputType::Full => {
+ //                           println!("{:?}", results);
+ //                       }
+ //                       OutputType::Raw => {
+ //                           let mut out = io::stdout();
+ //                           for result in results {
+ //                               out.write_all(result.original.as_ref()).unwrap();
+ //                           }
+ //                       }
+ //                       OutputType::Html => {
+ //                           for m in results {
+ //                               println!(
+ //                                   "{}",
+ //                                   m.body
+ //                                       .iter()
+ //                                       .filter(|x| x.mime == Mime::Html)
+ //                                       .collect::<Vec<&Body>>()
+ //                                       .first()
+ //                                       .map(|b| b.value.clone())
+ //                                       .unwrap_or_else(|| "No body".to_string())
+ //                               );
+ //                           }
+ //                       }
+ //                   }
                 }
                 Err(e) => error!("{}", e),
             }
@@ -124,46 +128,7 @@ pub async fn main() {
                 Ok(store) => {
                     let result = store.get_message(id);
                     match result {
-                        Ok(Some(good_msg)) => match output {
+                        Ok(Some(good_msg)) => println!("{}", good_msg.display(&output)),
-                            OutputType::Short => {
-                                println!("{} | {}", good_msg.id, good_msg.subject);
-                            }
-                            OutputType::Raw => {
-                                io::stdout().write_all(good_msg.original.as_ref()).unwrap();
-                            }
-                            OutputType::Full => {
-                                println!("From: {}", good_msg.from);
-                                println!(
-                                    "To: {}",
-                                    good_msg
-                                        .recipients
-                                        .get(0)
-                                        .unwrap_or(&String::from("Unknown"))
-                                );
-                                println!("Subject: {}", good_msg.subject);
-                                println!(
-                                    "{}",
-                                    good_msg
-                                        .body
-                                        .first()
-                                        .map(|b| b.value.clone())
-                                        .unwrap_or_else(|| "No body".to_string())
-                                );
-                            }
-                            OutputType::Html => {
-                                println!(
-                                    "{}",
-                                    good_msg
-                                        .body
-                                        .iter()
-                                        .filter(|x| x.mime == Mime::Html)
-                                        .collect::<Vec<&Body>>()
-                                        .first()
-                                        .map(|b| b.value.clone())
-                                        .unwrap_or_else(|| "No body".to_string())
-                                );
-                            }
-                        },
                         Ok(None) => error!("Message not found"),
                         Err(e) => error!("ERROR {}", e),
                     }
@@ -174,18 +139,18 @@ pub async fn main() {
         Command::Interactive {} => {
             terminal::start(index_dir_path).unwrap();
         }
-        Command::Latest { num: _num } => {
+        Command::Latest { num: _num, skip, output } => {
             let message_store = MessageStoreBuilder::new()
                 .storage(Storages::Tantivy(index_dir_path.clone()))
                 .searcher(Searchers::Tantivy(index_dir_path))
                 .build();
             match message_store {
                 Ok(store) => {
-                    let page = store.get_messages_page(0, _num);
+                    let page = store.get_messages_page(skip, _num);
                     match page {
                         Ok(msgs) => {
                             for m in msgs {
-                                println!("{}", m.id);
+                                println!("{}", m.display(&output));
                             }
                         }
                         Err(e) => println!("Could not read messages, {}", e),

src/cmd/mod.rs

@@ -1,8 +1,7 @@
 use std::ffi::OsStr;
 use std::path::PathBuf;
-use std::str::FromStr;
-use std::{error, fmt};
 use structopt::StructOpt;
+use crate::readmail::display::OutputType;
 
 pub fn expand_path(input: &OsStr) -> PathBuf {
     let input_str = input
@@ -14,49 +13,8 @@ pub fn expand_path(input: &OsStr) -> PathBuf {
     PathBuf::from(expanded)
 }
 
-#[derive(Debug)]
-pub enum OutputType {
-    Short,
-    Full,
-    Raw,
-    Html,
-}
 
-#[derive(Debug)]
-pub enum OutputTypeError {
-    UnknownTypeError,
-}
 
-impl fmt::Display for OutputTypeError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Could not figure out output format")
-    }
-}
-
-// This is important for other errors to wrap this one.
-impl std::error::Error for OutputTypeError {
-    fn description(&self) -> &str {
-        "invalid first item to double"
-    }
-
-    fn cause(&self) -> Option<&dyn error::Error> {
-        // Generic error, underlying cause isn't tracked.
-        None
-    }
-}
-
-impl FromStr for OutputType {
-    type Err = OutputTypeError;
-    fn from_str(input: &str) -> Result<OutputType, Self::Err> {
-        match input.to_lowercase().as_str() {
-            "short" => Ok(OutputType::Short),
-            "full" => Ok(OutputType::Full),
-            "raw" => Ok(OutputType::Raw),
-            "html" => Ok(OutputType::Html),
-            _ => Err(OutputTypeError::UnknownTypeError),
-        }
-    }
-}
 
 #[derive(Debug, StructOpt)]
 #[structopt(
@@ -115,6 +73,9 @@ pub enum Command {
 
         #[structopt(short, long, default_value = "100")]
         num: usize,
+
+        #[structopt(short, long)]
+        advanced: bool,
     },
     #[structopt(name = "date", rename_all = "kebab-case")]
     Date { term: i64 },
@@ -131,6 +92,10 @@ pub enum Command {
     Latest {
         #[structopt(short, long)]
         num: usize,
+        #[structopt(short, long, default_value = "0")]
+        skip: usize,
+        #[structopt(short, long, default_value = "short")]
+        output: OutputType,
     },
 
     #[structopt(name = "tag")]

src/message/mod.rs

@@ -8,7 +8,8 @@ use sha2::{Digest, Sha512};
 use std::collections::HashSet;
 use std::convert::AsRef;
 use std::fmt;
-use std::string::ToString;
+use crate::readmail::display::{DisplayAs, OutputType};
+
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Mime {
@@ -106,16 +107,21 @@ pub fn get_id(data: &[u8]) -> String {
     format!("{:x}", Sha512::digest(data))
 }
 
-impl ToString for ShortMessage {
+impl fmt::Display for ShortMessage {
-    fn to_string(&self) -> String {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let dt = Local.timestamp(self.date as i64, 0);
         let dstr = dt.format("%a %b %e %T %Y").to_string();
-        format!("{}: [{}] {}", dstr, self.from, self.subject.as_str())
+        write!(f, "{}: [{}] {}", dstr, self.from, self.subject.as_str())
     }
 }
 
-impl ToString for Message {
+impl fmt::Display for Message {
-    fn to_string(&self) -> String {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.display(&OutputType::Short))
+    }
+}
+impl DisplayAs for Message {
+    fn display(&self, t: &OutputType) -> String {
         let dt = Local.timestamp(self.date as i64, 0);
         let dstr = dt.format("%a %b %e %T %Y").to_string();
         let tags = if self.tags.is_empty() {
@@ -123,13 +129,28 @@ impl ToString for Message {
         } else {
             String::from("")
         };
-        format!(
+        match t {
-            "{} {}: [{}] {}",
+            OutputType::Short => format!("{} | {} | {}", self.short_id(), dstr, self.subject.as_str()),
-            tags,
+            OutputType::Full => format!(
-            dstr,
+            r#"
+        From: {}
+        to/cc/bcc: {}
+        Date: {}
+        Subject: {}
+
+        {} 
+        # {} 
+        "#,
             self.from,
-            self.subject.as_str()
+            self.recipients.join(","),
-        )
+            dstr,
+            self.subject,
+self.get_body(None).as_text(), self.id
+        ),
+            OutputType::Html => format!("{}", self.get_body(Some(Mime::Html)).as_text()),
+            OutputType::Summary => format!("{} | {} [{}]", dstr, self.subject.as_str(), self.from),
+            OutputType::Raw => String::from_utf8(self.original.clone()).unwrap_or(String::from("BAD FILE, please open an issue")),
+        }
     }
 }
 
@@ -146,7 +167,8 @@ impl MessageError {
 }
 
 impl Message {
-    pub fn from_parsedmail(msg: &ParsedMail, id: String) -> Result<Self, MessageError> {
+    pub fn from_parsedmail(msg: &ParsedMail) -> Result<Self, MessageError> {
+        let id = get_id(msg.data);
         let original = Vec::from(msg.data);
         let headers = &msg.headers;
         let mut subject: String = "".to_string();
@@ -193,42 +215,26 @@ impl Message {
         })
     }
     pub fn from_data(data: Vec<u8>) -> Result<Self, MessageError> {
-        let id = get_id(data.as_ref());
         let parsed_mail = parse_mail(data.as_slice()).map_err(|_| MessageError {
             message: String::from("Unable to parse email data"),
         })?;
-        Self::from_parsedmail(&parsed_mail, id)
+        Self::from_parsedmail(&parsed_mail)
     }
     pub fn from_mailentry(mut mailentry: MailEntry) -> Result<Self, MessageError> {
-        let id = mailentry.id().to_owned();
         match mailentry.parsed() {
-            Ok(parsed) => Self::from_parsedmail(&parsed, id),
+            Ok(parsed) => Self::from_parsedmail(&parsed),
             Err(_) => Err(MessageError {
-                message: format!("Failed to parse email id {}", id),
+                message: format!("Failed to parse email id {}", mailentry.id()),
             }),
         }
     }
-    pub fn get_body(&self) -> &Body {
+    pub fn get_body(&self, mime: Option<Mime>) -> &Body {
-        self.body.get(0).unwrap()
+        let m = mime.unwrap_or(Mime::PlainText);
+        self.body.iter().find(|b| b.mime == m ).unwrap_or(self.body.get(0).unwrap())
     }
-    pub fn to_long_string(&self) -> String {
-        format!(
-            r#"
-        From: {}
-        to/cc/bcc: {}
-        Subject: {}
 
-        {} 
+    pub fn short_id(&self) -> &str{
-        "#,
+        &self.id[..24]
-            self.from,
-            self.recipients.join(","),
-            self.subject,
-            self.body
-                .iter()
-                .map(|b| b.value.replace('\r', ""))
-                .collect::<Vec<String>>()
-                .join("-----")
-        )
     }
 }
 #[allow(dead_code)]

src/readmail/display.rs

@@ -0,0 +1,64 @@
+use std::fmt;
+
+impl fmt::Display for OutputTypeError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Could not figure out output format")
+    }
+}
+
+// This is important for other errors to wrap this one.
+impl std::error::Error for OutputTypeError {
+    fn description(&self) -> &str {
+        "invalid first item to double"
+    }
+
+    fn cause(&self) -> Option<&dyn std::error::Error> {
+        // Generic error, underlying cause isn't tracked.
+        None
+    }
+}
+
+impl std::str::FromStr for OutputType {
+    type Err = OutputTypeError;
+    fn from_str(input: &str) -> Result<OutputType, Self::Err> {
+        match input.to_lowercase().as_str() {
+            "short" => Ok(OutputType::Short),
+            "full" => Ok(OutputType::Full),
+            "raw" => Ok(OutputType::Raw),
+            "html" => Ok(OutputType::Html),
+            "summary" => Ok(OutputType::Summary),
+            _ => Err(OutputTypeError::UnknownTypeError),
+        }
+    }
+}
+impl fmt::Display for OutputType{
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let msg = match self {
+            OutputType::Summary => "Summary",
+            OutputType::Full => "Full",
+            OutputType::Html => "Html",
+            OutputType::Raw => "Raw",
+            OutputType::Short  => "Short",
+        };
+        write!(f,"{}", msg)
+    }
+}
+
+#[derive(Debug)]
+pub enum OutputTypeError {
+    UnknownTypeError,
+}
+
+#[derive(Debug)]
+pub enum OutputType {
+    Summary,
+    Short,
+    Full,
+    Raw,
+    Html,
+}
+
+
+pub trait DisplayAs {
+    fn display(&self, t: &OutputType) -> String;
+}

src/readmail/mod.rs

@@ -3,9 +3,11 @@ use crate::message::{Body, Mime};
 use log::debug;
 use mailparse::*;
 use select::document::Document;
-use select::predicate::Text;
+use select::predicate::{Text, Name, Predicate};
 use std::cmp::Ordering;
 
+pub mod display;
+
 fn cmp_body(x: &Body, y: &Body, prefer: &Mime) -> Ordering {
     if x.mime == y.mime {
         return x.value.len().cmp(&y.value.len());
@@ -66,8 +68,8 @@ pub fn extract_body(msg: &ParsedMail, prefer_html: bool) -> Vec<Body> {
 
 pub fn html2text(text: &str) -> String {
     let document = Document::from(text);
-    let text_nodes = document
+    let body = document.find(Name("body")).nth(0).unwrap();
-        .find(Text)
+    let text_nodes = body.find(Text)
         .map(|x| String::from(x.text().trim()))
         .filter(|x| x.len() > 1)
         .collect::<Vec<String>>();

src/stores/_impl/tantivy.rs

@@ -1,15 +1,15 @@
 use crate::message::{Message, MessageError};
 use crate::stores::{IMessageSearcher, MessageStoreError};
-use log::info;
+use log::{error, info};
 use std::cmp;
 use std::collections::HashSet;
 use std::fs;
 use std::panic;
 use std::path::PathBuf;
 use std::string::ToString;
-use tantivy::collector::TopDocs;
+use tantivy::collector::{TopDocs, Count};
 use tantivy::directory::MmapDirectory;
-use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, TermQuery};
+use tantivy::query::{AllQuery, BooleanQuery, FuzzyTermQuery, Occur, Query, TermQuery, RegexQuery};
 use tantivy::schema::*;
 const BYTES_IN_MB: usize = 1024 * 1024;
 
@@ -218,6 +218,7 @@ impl TantivyStore {
                     .iter()
                     .for_each(|t| document.add_text(email.tag, t.as_str()));
                 indexer.add_document(document);
+                indexer.commit();
                 Ok(msg.id)
             }
             None => Err(MessageStoreError::CouldNotAddMessage(
@@ -305,17 +306,14 @@ impl TantivyStore {
     ) -> Result<Vec<TantivyMessage>, MessageStoreError> {
         let searcher = self.reader.searcher();
         let skip = _skip.unwrap_or(0);
-        let mut docs = searcher
+        let docs = searcher
             .search(
                 &AllQuery,
-                &TopDocs::with_limit(num + skip).order_by_u64_field(self.email.date),
+                &TopDocs::with_limit(num).and_offset(skip).order_by_u64_field(self.email.date),
             )
             .map_err(|_| MessageStoreError::CouldNotGetMessages(vec![]))?;
         let mut ret = vec![];
-        let page = docs
+        for doc in docs {
-            .drain(skip..)
-            .collect::<Vec<(u64, tantivy::DocAddress)>>();
-        for doc in page {
             let retrieved_doc = searcher.doc(doc.1).unwrap();
             ret.push(
                 TantivyMessage::from_tantivy(retrieved_doc, &self.email).map_err(|_| {
@@ -329,17 +327,17 @@ impl TantivyStore {
     pub fn get_doc(&self, id: &str) -> Result<Document, tantivy::TantivyError> {
         // Is this needed? self.reader.load_searchers()?;
         let searcher = self.reader.searcher();
-        let termq = TermQuery::new(
+        let termq = RegexQuery::from_pattern(format!("{}.*",id).as_str(), self.email.id)?;
-            Term::from_field_text(self.email.id, id),
+        let res = searcher.search(&termq, &(TopDocs::with_limit(1), Count));
-            IndexRecordOption::Basic,
+        match res {
-        );
+            Ok((doc, count)) => match doc.first() {
-        let addr = searcher.search(&termq, &TopDocs::with_limit(1));
-        match addr {
-            Ok(doc) => match doc.first() {
                 Some((_score, doc_address)) => searcher.doc(*doc_address),
-                None => Err(tantivy::TantivyError::InvalidArgument(
+                None => {
+                    error!("Got count {:}", count);
+                    Err(tantivy::TantivyError::InvalidArgument(
                     "Document not found".to_string(),
-                )),
+                ))
+                },
             },
 
             Err(e) => Err(e),

src/stores/message_store.rs

@@ -1,22 +1,24 @@
 use crate::message::Message;
 use crate::stores::{IMessageSearcher, IMessageStorage, IMessageStore, MessageStoreError};
+use async_trait::async_trait;
 use chrono::{DateTime, Utc};
+use futures::future::join_all;
 use log::error;
 use maildir::{MailEntry, Maildir};
 use pbr::{MultiBar, ProgressBar};
-use rayon::prelude::*;
+use std::{collections::HashSet, fmt};
-use std::collections::HashSet;
 use std::path::PathBuf;
-use std::sync::mpsc;
 use std::thread;
 use std::time::Duration;
+use tokio::sync::mpsc;
 
 pub struct MessageStore {
-    pub searcher: Box<dyn IMessageSearcher>,
+    pub searcher: Box<dyn IMessageSearcher + Send + Sync>,
-    pub storage: Option<Box<dyn IMessageStorage>>,
+    pub storage: Option<Box<dyn IMessageStorage + Send + Sync>>,
     progress: Option<ProgressBar<pbr::Pipe>>,
     display_progress: bool,
 }
+#[async_trait]
 impl IMessageStore for MessageStore {
     fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError> {
         self.searcher.get_message(id)
@@ -32,8 +34,8 @@ impl IMessageStore for MessageStore {
         Ok(id)
     }
 
-    fn add_maildir(&mut self, path: PathBuf, all: bool) -> Result<usize, MessageStoreError> {
+    async fn add_maildir(&mut self, path: PathBuf, all: bool) -> Result<usize, MessageStoreError> {
-        self.index_mails(path, all)
+        self.index_mails(path, all).await
     }
     fn tag_message_id(
         &mut self,
@@ -76,10 +78,21 @@ impl IMessageStore for MessageStore {
         unimplemented!();
     }
 }
+
+struct Entry {
+    e: MailEntry,
+}
+
+impl fmt::Debug for Entry {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Entry").field("e", &String::from("ARGH")).finish()
+    }
+}
+
 impl MessageStore {
     pub fn new(
-        searcher: Box<dyn IMessageSearcher>,
+        searcher: Box<dyn IMessageSearcher + Send + Sync>,
-        storage: Option<Box<dyn IMessageStorage>>,
+        storage: Option<Box<dyn IMessageStorage + Send + Sync>>,
         display_progress: bool,
     ) -> Self {
         MessageStore {
@@ -135,7 +148,26 @@ impl MessageStore {
         }
     }
 
-    fn do_index_mails(&mut self, maildir: Maildir, full: bool) -> Result<usize, MessageStoreError> {
+    fn parse_message(mail: MailEntry) -> Result<(Message, String), MessageStoreError> {
+        let message = Message::from_mailentry(mail);
+        match message {
+            Ok(msg) => {
+                let parsed_body = msg.get_body(None).as_text();
+                Ok((msg, parsed_body))
+            }
+            Err(err) => {
+                error!("A message could not be parsed: {}", err.message);
+                Err(MessageStoreError::CouldNotAddMessage(
+                    "Failed to parse email".to_string(),
+                ))
+            }
+        }
+    }
+    async fn do_index_mails(
+        &mut self,
+        maildir: Maildir,
+        full: bool,
+    ) -> Result<usize, MessageStoreError> {
         let mails: Vec<Result<MailEntry, _>> = Self::mail_iterator(&maildir, full).collect();
         let count = mails.len();
         self.start_indexing_process(count)?;
@@ -143,37 +175,32 @@ impl MessageStore {
         if self.display_progress {
             progress_handle = Some(self.init_progress(count));
         }
-        let (tx, rx) = mpsc::channel();
+        let (tx, mut rx) = mpsc::channel(100);
-
+        let handles = mails
-        let t = thread::spawn(move || {
+            .into_iter()
-            mails
+            .map(|m| {
-                .into_par_iter()
+                let tx = tx.clone();
-                .for_each_with(tx, |tx, msg| match msg {
+                tokio::spawn(async move {
-                    Ok(unparsed_msg) => {
+                    if let Ok(entry) = m {
-                        let message = Message::from_mailentry(unparsed_msg);
+                        let id = match entry.is_seen() {
-                        match message {
+                            true => None,
-                            Ok(msg) => {
+                            false => Some(String::from(entry.id()))
-                                let parsed_body = msg.get_body().as_text();
+                        };
-                                tx.send((msg, parsed_body))
+                        if let Ok((msg, body)) = MessageStore::parse_message(entry) {
-                                    .expect("Could not send to channel?")
+                            tx.send((msg, body, id)).await.unwrap();
+                        };
                     }
-                            Err(err) => {
+                })
-                                error!("A message could not be parsed: {}", err.message);
+            })
-                            }
+            .collect::<Vec<tokio::task::JoinHandle<_>>>();
-                        }
+        drop(tx);
-                    }
+        while let Some((msg, parsed_body, id)) = rx.recv().await {
-                    Err(e) => {
-                        error!("Failed to get message {}", e);
-                    }
-                });
-        });
-
-        while let Ok((msg, parsed_body)) = rx.recv() {
             self.add_message(msg, parsed_body)?;
             self.inc_progress();
+            id.map( |id| maildir.move_new_to_cur(&id).map_err(|_| MessageStoreError::CouldNotModifyMessage(format!("Message couldn't be moved {}", id))));
         }
+        join_all(handles).await;
         self.finish_indexing_process()?;
-        t.join().expect("Unable to join threads for some reason");
         self.finish_progress();
         if let Some(handle) = progress_handle {
             handle
@@ -183,11 +210,15 @@ impl MessageStore {
         Ok(count)
     }
 
-    pub fn index_mails(&mut self, path: PathBuf, full: bool) -> Result<usize, MessageStoreError> {
+    pub async fn index_mails(
+        &mut self,
+        path: PathBuf,
+        full: bool,
+    ) -> Result<usize, MessageStoreError> {
         let maildir = self.maildir(path);
         match maildir {
             Ok(maildir) => {
-                self.do_index_mails(maildir, full)?;
+                self.do_index_mails(maildir, full).await?;
                 Ok(1)
             }
             Err(_) => Err(MessageStoreError::CouldNotOpenMaildir(

src/stores/mod.rs

@@ -1,4 +1,5 @@
 use crate::message::Message;
+use async_trait::async_trait;
 use chrono::{DateTime, Utc};
 use std::collections::HashSet;
 use std::path::PathBuf;
@@ -192,6 +193,7 @@ pub trait IMessageStorage {
     ) -> Result<Vec<Message>, MessageStoreError>;
 }
 
+#[async_trait]
 pub trait IMessageStore {
     fn get_message(&self, id: String) -> Result<Option<Message>, MessageStoreError>;
     fn add_message(
@@ -199,7 +201,9 @@ pub trait IMessageStore {
         msg: Message,
         parsed_body: String,
     ) -> Result<String, MessageStoreError>;
-    fn add_maildir(&mut self, path: PathBuf, all: bool) -> Result<usize, MessageStoreError>;
+    
+    async fn add_maildir(&mut self, path: PathBuf, all: bool) -> Result<usize, MessageStoreError>;
+
     fn tag_message_id(
         &mut self,
         id: String,

src/terminal/store/list.rs

@@ -6,7 +6,6 @@ pub struct ListStore<'a> {
     pub selected: usize,
     pub page_size: usize,
     pub curr_idx: usize,
-    pub fetched_first: bool,
     pub message_store: &'a dyn IMessageStore,
 }
 
@@ -15,7 +14,6 @@ impl<'a> ListStore<'a> {
         ListStore {
             messages: vec![],
             selected: 0,
-            fetched_first: false,
             page_size: 10,
             curr_idx: 0,
             message_store: msg_store,
@@ -49,6 +47,8 @@ impl<'a> ListStore<'a> {
         if r < 0 {
             r = 0
         } else if r > l - 1 {
+            let mut messages = self.message_store.get_messages_page(r as usize, self.page_size);
+            self.messages.append(messages.as_mut().ok().unwrap());
             r = l
         };
         self.selected = r as usize;
@@ -56,14 +56,9 @@ impl<'a> ListStore<'a> {
     }
 
     pub fn latest(&mut self) {
-        let mut page_size = self.page_size;
-        if !self.fetched_first {
-            page_size = 1000;
-            self.fetched_first = true;
-        }
         let messages = self
             .message_store
-            .get_messages_page(self.curr_idx, page_size);
+            .get_messages_page(self.curr_idx, self.page_size);
         match messages {
             Ok(messages) => self.messages = messages,
             Err(_) => self.messages = vec![], // TODO Handle error

src/terminal/views/email_read.rs

@@ -1,4 +1,5 @@
 use crate::message::Message;
+use crate::readmail::display::{OutputType, DisplayAs};
 use tui::backend::Backend;
 use tui::layout::Rect;
 use tui::text::Text;
@@ -6,7 +7,7 @@ use tui::widgets::{Block, Borders, Paragraph, Wrap};
 use tui::Frame;
 
 pub fn draw<B: Backend>(f: &mut Frame<B>, message: &Message, scroll: u16) {
-    let text = message.to_long_string();
+    let text = message.display(&OutputType::Full);
     let f_r = f.size();
     let rect = Rect {
         x: f_r.x + f_r.width / 2 - 40,

src/terminal/views/search_results.rs

@@ -1,4 +1,5 @@
 use crate::terminal::store::Store;
+use crate::readmail::display::{OutputType, DisplayAs};
 use tui::backend::Backend;
 use tui::layout::Rect;
 use tui::style::{Color, Modifier, Style};
@@ -17,8 +18,7 @@ pub fn draw<B: Backend>(f: &mut Frame<B>, area: Rect, store: &Store) {
     let items: Vec<ListItem> = display
         .iter()
         .map(|s| {
-            let s = s.to_string();
+            ListItem::new(Span::raw(s.display(&OutputType::Summary)))
-            ListItem::new(Span::raw(s))
         })
         .collect::<Vec<ListItem>>();
     let list = List::new(items)