mod cache; mod database; pub mod error; mod store; use async_std::{ fs::{create_dir, read, remove_file, write}, path::Path, sync::{Arc, Mutex}, task::{sleep, spawn}, }; use database::Database; use error::{DBError, ErrorCode}; use rand::{distributions::Alphanumeric, thread_rng, Rng}; use std::{ collections::HashMap, fmt, slice, str, time::{Duration, Instant}, }; use store::Store; const DATA: &str = "data"; const ENTRY: &str = "databases"; trait FileData { fn to_bytes(&self) -> Vec; fn from_bytes(data: &mut slice::Iter) -> Result; } trait SessionData { fn add(&mut self, key: &str, value: &str, data: &str) -> Result, DBError>; fn eq(&self, key: &str, value: &str) -> Result, DBError>; fn list(&self, keys: Vec<&str>) -> Result, DBError>; } #[derive(Clone)] pub enum CacheType { Raw(String), DBMap(Store), TableMap, } impl CacheType { pub fn entry_type(&self) -> String { match self { CacheType::Raw(_) => "Raw".to_string(), CacheType::DBMap(_) => "DBMap".to_string(), CacheType::TableMap => "TableMap".to_string(), } } pub fn to_bytes(&self) -> Vec { let mut output = self.entry_type().into_bytes(); output.push(0); match self { CacheType::Raw(s) => output.append(&mut s.as_bytes().to_vec()), CacheType::DBMap(_) => (), CacheType::TableMap => (), } return output; } pub fn from_bytes(data: Vec) -> Result { let mut data_iter = data.iter(); let mut letter: u8; match data_iter.next() { Some(item) => letter = *item, None => return Err(DBError::new("empty file")), } let mut header: Vec = Vec::new(); while letter != 0 { header.push(letter.clone()); match data_iter.next() { Some(item) => letter = *item, None => return Err(DBError::new("incomplete file")), } } let header = str::from_utf8(&header).unwrap().to_string(); match header.as_str() { "Raw" => { let mut output: Vec = Vec::new(); for letter in data_iter { output.push(letter.clone()); } Ok(CacheType::Raw(str::from_utf8(&output).unwrap().to_string())) } "DBMap" => Ok(CacheType::DBMap(Store::new())), "TableMap" => Ok(CacheType::TableMap), _ => Err(DBError::new("data corruption")), } } } impl fmt::Display for CacheType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { CacheType::Raw(s) => write!(f, "{}", s), CacheType::DBMap(_) => todo!(), CacheType::TableMap => todo!(), } } } #[derive(Clone)] struct CacheEntry { data: CacheType, last_used: Instant, } impl CacheEntry { fn new(data: CacheType) -> Self { Self { data: data, last_used: Instant::now(), } } fn elapsed(&self) -> Duration { self.last_used.elapsed() } fn touch(&mut self) { self.last_used = Instant::now(); } fn update(&mut self, data: CacheType) { self.data = data; self.touch(); } } impl fmt::Display for CacheEntry { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.data) } } #[derive(Clone)] pub struct MoreThanText { cache: Arc>>, dir: String, session: Vec, } impl MoreThanText { pub async fn new(dir: &str) -> Result { let data_dir = Path::new(dir).join(DATA); if !data_dir.is_dir().await { match create_dir(&data_dir).await { Ok(_) => (), Err(err) => { let mut error = DBError::new("failed to create data directory"); error.add_source(err); return Err(error); } } } let mut output = Self { cache: Arc::new(Mutex::new(HashMap::new())), dir: data_dir.to_str().unwrap().to_string(), session: Vec::new(), }; let entry_file = Path::new(dir).join(ENTRY); let id: String; if entry_file.is_file().await { let holder = read(entry_file).await.unwrap(); id = str::from_utf8(&holder).unwrap().to_string(); } else { id = output .add_entry(CacheType::DBMap(Store::new())) .await .unwrap(); write(entry_file, id.as_bytes()).await.unwrap(); } output.session.push(id); let looper = output.cache.clone(); spawn(async move { let hold_time = Duration::from_secs(300); loop { sleep(Duration::from_secs(1)).await; let mut ids: Vec = Vec::new(); let mut cache = looper.lock().await; for (id, entry) in cache.iter() { if entry.elapsed() > hold_time { ids.push(id.to_string()); } } for id in ids.iter() { cache.remove(id); } } }); Ok(output) } fn filename(&self, id: &str) -> String { let filename = Path::new(&self.dir).join(&id); filename.into_os_string().into_string().unwrap() } fn new_id(&self) -> String { thread_rng().sample_iter(&Alphanumeric).take(64).collect() } async fn add(&self, feature: &str, key: &str, value: &str) -> Self { let mut ids: Vec = Vec::new(); for id in self.session.clone().into_iter() { let holder = self.get_entry(&id).await.unwrap(); //holder.add(feature, key, value); } let mut output = self.clone(); output.session.clear(); output.session.push(value.to_string()); output } async fn list(&self, feature: Vec<&str>) -> Result, DBError> { Ok(Vec::new()) } async fn add_entry(&self, entry: CacheType) -> Result { let mut id: String = "".to_string(); let mut dup = true; while dup { id = thread_rng().sample_iter(&Alphanumeric).take(32).collect(); dup = Path::new(&self.dir).join(&id).as_path().exists().await; } match write(Path::new(&self.filename(&id)), entry.to_bytes()).await { Ok(_) => (), Err(err) => { let mut error = DBError::new("data write"); error.add_source(err); return Err(error); } }; let mut cache = self.cache.lock().await; let data = CacheEntry::new(entry); cache.insert(id.clone(), data); Ok(id) } async fn get_entry(&self, id: &str) -> Result { let mut cache = self.cache.lock().await; match cache.get_mut(id) { Some(entry) => { entry.touch(); Ok(entry.clone()) } None => match read(Path::new(&self.filename(id))).await { Ok(content) => { let data = CacheEntry::new(CacheType::from_bytes(content).unwrap()); cache.insert(id.to_string(), data.clone()); Ok(data) } Err(_) => Err(DBError::new("cache entry not found")), }, } } async fn update_entry(&self, id: &str, entry: CacheType) -> Result<(), DBError> { match self.get_entry(id).await { Ok(_) => (), Err(err) => return Err(err), } match write(Path::new(&self.filename(id)), entry.to_bytes()).await { Ok(_) => (), Err(err) => { let mut error = DBError::new("data write"); error.add_source(err); return Err(error); } } let mut cache = self.cache.lock().await; let data = CacheEntry::new(entry); cache.insert(id.to_string(), data); Ok(()) } async fn delete_entry(&self, id: &str) -> Result<(), DBError> { let mut cache = self.cache.lock().await; cache.remove(id); match remove_file(Path::new(&self.filename(id))).await { Ok(_) => Ok(()), Err(err) => { let mut error = DBError::new("data delete"); error.add_source(err); Err(error) } } } } #[cfg(test)] mod setup { use super::*; use async_std::fs::remove_dir_all; use tempfile::{tempdir, TempDir}; pub struct MTT { pub db: MoreThanText, pub dir: TempDir, } impl MTT { pub async fn new() -> Self { let dir = tempdir().unwrap(); let db = MoreThanText::new(dir.path().to_str().unwrap()) .await .unwrap(); Self { db: db, dir: dir } } pub async fn create_io_error(&self) { remove_dir_all(self.dir.path().join(DATA)).await.unwrap(); } } } #[cfg(test)] mod init { use super::*; use std::error::Error; use tempfile::tempdir; #[async_std::test] async fn create_data_dir() { let dir = tempdir().unwrap(); MoreThanText::new(dir.path().to_str().unwrap()) .await .unwrap(); let data_dir = dir.path().join(DATA); assert!(data_dir.is_dir(), "Did not create the data directory."); dir.close().unwrap(); } #[async_std::test] async fn existing_data_dir() { let dir = tempdir().unwrap(); let data_dir = dir.path().join(DATA); create_dir(data_dir).await.unwrap(); MoreThanText::new(dir.path().to_str().unwrap()) .await .unwrap(); dir.close().unwrap(); } #[async_std::test] async fn bad_data_dir() { let msg = "could not create directory"; match MoreThanText::new("kljsdgfhslkfrh").await { Ok(_) => assert!(false, "This test should fail to create a data directory"), Err(err) => { assert_eq!(err.to_string(), "failed to create data directory"); assert!(err.source().is_some(), "Must include the source error."); let err_msg = err.source().unwrap().to_string(); assert!(err_msg.contains(msg), "'{}' not in '{}'", msg, err_msg); } }; } #[async_std::test] async fn creates_entry_point() { let dir = tempdir().unwrap(); let db = MoreThanText::new(dir.path().to_str().unwrap()) .await .unwrap(); let entry = dir.path().join(ENTRY); assert!(entry.is_file(), "Did not create entry point file."); let data = read(entry).await.unwrap(); let id = str::from_utf8(&data).unwrap(); let cache = db.get_entry(&id).await.unwrap(); assert_eq!(cache.data.entry_type(), "DBMap"); assert_eq!(db.session, [id]); } #[async_std::test] async fn use_existing_entry_point() { let dir = tempdir().unwrap(); let db1 = MoreThanText::new(dir.path().to_str().unwrap()) .await .unwrap(); let db2 = MoreThanText::new(dir.path().to_str().unwrap()) .await .unwrap(); assert_eq!(db1.session, db2.session, "Did not read existing entry."); } } #[cfg(test)] mod data { use super::*; use setup::MTT; #[async_std::test] async fn ids_are_random() { let mtt = MTT::new().await; let id1 = mtt.db.new_id(); let id2 = mtt.db.new_id(); assert_ne!(id1, id2, "Ids should be random"); } #[async_std::test] async fn add_database() { let mtt = MTT::new().await; let name = "fred"; let id = "*gsdfg"; let output = mtt.db.add("database", name, id).await; assert_eq!(output.session, [id], "should update session info."); /* assert_eq!( mtt.db.list(["database"].to_vec()).await.unwrap(), [name], "Should list the databases." ); */ } } #[cfg(test)] mod cache_test { use super::*; use async_std::fs::read; use setup::MTT; use std::error::Error; #[async_std::test] async fn entry_ids_are_random() { let mtt = MTT::new().await; let data1 = CacheType::Raw("one".to_string()); let data2 = CacheType::Raw("two".to_string()); let id1 = mtt.db.add_entry(data1).await.unwrap(); let id2 = mtt.db.add_entry(data2).await.unwrap(); assert_ne!(id1, id2, "Ids should be unique.") } #[async_std::test] async fn store_cache() { let mtt = MTT::new().await; let data = "something"; let expected = CacheType::Raw(data.to_string()); let id = mtt.db.add_entry(expected.clone()).await.unwrap(); let output = mtt.db.get_entry(&id).await.unwrap(); assert_eq!(output.to_string(), data); let dfile = mtt.dir.path().join(DATA).join(&id); assert!(dfile.is_file(), "Cache file should exist."); let content = read(dfile).await.unwrap(); assert_eq!(content, expected.to_bytes()); } #[async_std::test] async fn get_entry_uodates_time() { let mtt = MTT::new().await; let id = "something"; let holder = CacheEntry { data: CacheType::Raw("old".to_string()), last_used: Instant::now() - Duration::from_secs(200), }; let mut cache = mtt.db.cache.lock().await; cache.insert(id.to_string(), holder); drop(cache); mtt.db.get_entry(&id).await.unwrap(); let cache = mtt.db.cache.lock().await; let entry = cache.get(id).unwrap(); let held = entry.elapsed(); assert!( Duration::from_secs(1) > held, "Duration was {:?}, should have been close to 0s.", held ); } #[async_std::test] async fn retrieve_from_disk() { let mtt = MTT::new().await; let id = "someid"; let data = CacheType::Raw("stored".to_string()); write(mtt.dir.path().join(DATA).join(id), data.to_bytes()) .await .unwrap(); let output = mtt.db.get_entry(id).await.unwrap(); assert_eq!(output.to_string(), data.to_string()); let cache = mtt.db.cache.lock().await; let stored = cache.get(id); assert!(stored.is_some(), "Did not store entry in the cache."); } #[async_std::test] async fn store_bad_file() { let mtt = MTT::new().await; let msg = "could not write to file"; mtt.create_io_error().await; match mtt.db.add_entry(CacheType::Raw("fail".to_string())).await { Ok(_) => assert!(false, "This test should fail."), Err(err) => { assert_eq!(err.to_string(), "data write"); assert!(err.source().is_some(), "Must include the source error."); let err_msg = err.source().unwrap().to_string(); assert!(err_msg.contains(msg), "'{}' not in '{}'", msg, err_msg); } } } #[async_std::test] async fn retrieve_bad_id() { let mtt = MTT::new().await; match mtt.db.get_entry(&"Not Valid").await { Ok(_) => assert!(false, "Should have raised an error."), Err(err) => assert_eq!(err.to_string(), "cache entry not found"), } } #[async_std::test] async fn update_cache_entry() { let mtt = MTT::new().await; let id = "updateable"; let holder = CacheEntry { data: CacheType::Raw("elder".to_string()), last_used: Instant::now() - Duration::from_secs(500), }; let mut cache = mtt.db.cache.lock().await; cache.insert(id.to_string(), holder); drop(cache); let expected = "different"; let expect = CacheType::Raw(expected.to_string()); mtt.db.update_entry(id, expect.clone()).await.unwrap(); let output = mtt.db.get_entry(id).await.unwrap(); assert_eq!(output.to_string(), expected); let cache = mtt.db.cache.lock().await; let entry = cache.get(id).unwrap(); let held = entry.elapsed(); assert!( Duration::from_secs(1) > held, "Duration was {:?}, should have been close to 0s.", held ); drop(cache); let content = read(mtt.dir.path().join(DATA).join(id)).await.unwrap(); assert_eq!(content, expect.to_bytes()); } #[async_std::test] async fn update_bad_id() { let mtt = MTT::new().await; match mtt .db .update_entry("wilma", CacheType::Raw("wrong".to_string())) .await { Ok(_) => assert!(false, "Bad id should raise an error."), Err(err) => assert_eq!(err.to_string(), "cache entry not found"), } } #[async_std::test] async fn update_bad_file() { let mtt = MTT::new().await; let msg = "could not write to file"; let id = mtt .db .add_entry(CacheType::Raw("fleeting".to_string())) .await .unwrap(); mtt.create_io_error().await; match mtt .db .update_entry(&id, CacheType::Raw("failure".to_string())) .await { Ok(_) => assert!(false, "This should produce a write failure."), Err(err) => { assert_eq!(err.to_string(), "data write"); assert!(err.source().is_some(), "Must include the source error."); let err_msg = err.source().unwrap().to_string(); assert!(err_msg.contains(msg), "'{}' not in '{}'", msg, err_msg); } } } #[async_std::test] async fn remove_entry() { let mtt = MTT::new().await; let id = mtt .db .add_entry(CacheType::Raw("delete".to_string())) .await .unwrap(); mtt.db.delete_entry(&id).await.unwrap(); match mtt.db.get_entry(&id).await { Ok(_) => assert!(false, "Entry should be removed from cache."), Err(_) => (), }; } #[async_std::test] async fn remove_missing_entry() { let mtt = MTT::new().await; let msg = "could not remove file"; match mtt.db.delete_entry("missing").await { Ok(_) => assert!(false, "This should produce a write failure."), Err(err) => { assert_eq!(err.to_string(), "data delete"); assert!(err.source().is_some(), "Must include the source error."); let err_msg = err.source().unwrap().to_string(); assert!(err_msg.contains(msg), "'{}' not in '{}'", msg, err_msg); } } } #[async_std::test] async fn remove_older() { let mtt = MTT::new().await; let id = mtt .db .add_entry(CacheType::Raw("removed".to_string())) .await .unwrap(); let mut cache = mtt.db.cache.lock().await; let entry = cache.get_mut(&id).unwrap(); entry.last_used = Instant::now() - Duration::from_secs(1000); drop(cache); sleep(Duration::from_secs(2)).await; let cache = mtt.db.cache.lock().await; let output = cache.get(&id); assert!(output.is_none(), "The entry shoould not be in memory."); drop(cache); let filename = mtt.db.filename(&id); let fpath = Path::new(&filename); assert!( fpath.is_file().await, "The stored version should still exist." ); } #[async_std::test] async fn keep_newer() { let mtt = MTT::new().await; let id = mtt .db .add_entry(CacheType::Raw("keep".to_string())) .await .unwrap(); sleep(Duration::from_secs(2)).await; let cache = mtt.db.cache.lock().await; let output = cache.get(&id); assert!(output.is_some(), "The entry shoould be in memory."); } } #[cfg(test)] mod cache_entry { use super::*; #[test] fn init() { let text = "new entry"; let holder = CacheEntry::new(CacheType::Raw(text.to_string())); assert_eq!(holder.to_string(), text); let held = holder.elapsed(); assert!( Duration::from_secs(1) > held, "Duration was {:?}, should have been close to 0s.", held ); } #[test] fn older() { let secs = 800; let holder = CacheEntry { data: CacheType::Raw("older".to_string()), last_used: Instant::now() - Duration::from_secs(secs), }; let held = holder.elapsed() - Duration::from_secs(secs); assert!( Duration::from_secs(1) > held, "{:?} should be close to {}s", holder.elapsed(), secs ); } #[test] fn accessed() { let mut holder = CacheEntry { data: CacheType::Raw("older".to_string()), last_used: Instant::now() - Duration::from_secs(700), }; holder.touch(); let held = holder.elapsed(); assert!( Duration::from_secs(1) > held, "Duration was {:?}, should have been close to 0s.", held ); } #[test] fn updated() { let text = "new data"; let mut holder = CacheEntry { data: CacheType::Raw("old data".to_string()), last_used: Instant::now() - Duration::from_secs(900), }; holder.update(CacheType::Raw(text.to_string())); assert_eq!(holder.to_string(), text); let held = holder.elapsed(); assert!( Duration::from_secs(1) > held, "Duration was {:?}, should have been close to 0s.", held ); } } #[cfg(test)] mod enum_ctype { use super::*; #[test] fn bad_file_header() { let mut data: Vec = Vec::new(); let mut ctype = "jlksdfg".as_bytes().to_vec(); let mut cdata = "ghjk".as_bytes().to_vec(); data.append(&mut ctype); data.push(0); data.append(&mut cdata); match CacheType::from_bytes(data) { Ok(_) => assert!(false, "This should fail."), Err(err) => assert_eq!(err.to_string(), "data corruption"), } } #[test] fn incomplete_file() { let mut data: Vec = Vec::new(); let mut ctype = "uoisfde".as_bytes().to_vec(); data.append(&mut ctype); match CacheType::from_bytes(data) { Ok(_) => assert!(false, "This should fail."), Err(err) => assert_eq!(err.to_string(), "incomplete file"), } } #[test] fn empty_file() { let data: Vec = Vec::new(); match CacheType::from_bytes(data) { Ok(_) => assert!(false, "This should fail."), Err(err) => assert_eq!(err.to_string(), "empty file"), } } #[test] fn get_raw_type() { let holder = CacheType::Raw("nothing important".to_string()); assert_eq!(holder.entry_type(), "Raw"); } #[test] fn get_raw_bytes() { let data = "addams"; let holder = CacheType::Raw(data.to_string()); let mut expected = holder.entry_type().into_bytes(); expected.push(0); expected.append(&mut data.as_bytes().to_vec()); let output = holder.to_bytes(); assert_eq!(output, expected); } #[test] fn from_raw_bytes() { let holder = CacheType::Raw("stored item".to_string()); let data = holder.to_bytes(); let output = CacheType::from_bytes(data).unwrap(); assert_eq!(output.to_string(), holder.to_string()); } #[test] fn get_dbmap_type() { let holder = CacheType::DBMap(Store::new()); assert_eq!(holder.entry_type(), "DBMap"); } #[test] fn get_new_databases_bytes() { let holder = CacheType::DBMap(Store::new()); let mut expected = "DBMap".as_bytes().to_vec(); expected.push(0); let output = holder.to_bytes(); assert_eq!(output, expected); } #[test] fn from_new_databases_bytes() { let mut data = "DBMap".as_bytes().to_vec(); data.push(0); let output = CacheType::from_bytes(data).unwrap(); assert_eq!(output.entry_type(), "DBMap"); } #[test] fn get_tablemap_type() { let holder = CacheType::TableMap; assert_eq!(holder.entry_type(), "TableMap"); } #[test] fn get_new_database_bytes() { let holder = CacheType::TableMap; let mut expected = "TableMap".as_bytes().to_vec(); expected.push(0); let output = holder.to_bytes(); assert_eq!(output, expected); } #[test] fn from_new_database_bytes() { let mut data = "TableMap".as_bytes().to_vec(); data.push(0); let output = CacheType::from_bytes(data).unwrap(); assert_eq!(output.entry_type(), "TableMap"); } }