From 596899a31a6ce0524897f6553a14345bc4b9879d Mon Sep 17 00:00:00 2001 From: Paul Z Date: Thu, 20 Apr 2023 13:36:38 +0200 Subject: [PATCH] added ttl --- Cargo.lock | 84 ++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 ++ src/main.rs | 93 +++++++++++++++++++++++++++++++++++++++++++++++-- src/metadata.rs | 2 ++ src/openid.rs | 1 + 5 files changed, 178 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cbfe62..a4c9d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,10 +129,12 @@ name = "bin" version = "0.1.0" dependencies = [ "axum", + "chrono", "dotenvy", "futures-util", "markdown", "openidconnect", + "parse_duration", "rand", "render", "reqwest", @@ -195,9 +197,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", + "js-sys", "num-integer", "num-traits", "serde", + "time", + "wasm-bindgen", "winapi", ] @@ -521,7 +526,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -862,10 +867,35 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.2" @@ -883,6 +913,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -904,6 +944,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -1035,6 +1087,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "parse_duration" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d" +dependencies = [ + "lazy_static", + "num", + "regex", +] + [[package]] name = "pem-rfc7468" version = "0.6.0" @@ -1642,6 +1705,17 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1834,6 +1908,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 0bb5729..5c5d638 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,6 @@ rand = "0.8.5" dotenvy = "0.15" reqwest = { version="0.11", default_features=false} markdown = "0.3.0" +chrono = { version="0.4", features=["serde"]} +parse_duration = "2.1" + diff --git a/src/main.rs b/src/main.rs index 80d784b..65fa91b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,18 +1,27 @@ -use std::{collections::HashMap, env, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + env, + fmt::LowerExp, + str::FromStr, + sync::Arc, + time::Duration, +}; use axum::{ body::StreamBody, - extract::{BodyStream, Path, State}, + extract::{BodyStream, Path, Query, State}, headers::ContentType, http::{header, HeaderMap, StatusCode}, response::{Html, IntoResponse, Redirect}, routing::get, Router, TypedHeader, }; +use chrono::{NaiveDateTime, Utc}; use futures_util::StreamExt; use metadata::Metadata; use openid::Login; use render::{html, raw}; +use serde::Deserialize; use tokio::{ fs::{self, File}, io::{AsyncReadExt, AsyncWriteExt}, @@ -39,18 +48,22 @@ enum Error { DataFileExists, #[error("datafile without metafile")] DataFileWithoutMetaFile, + + #[error("could not parse ttl")] + ParseTtl, } type HandlerResult = Result; impl IntoResponse for Error { fn into_response(self) -> axum::response::Response { - println!("main error: {:?}", self); match self { Self::ItemNotFound => (StatusCode::NOT_FOUND, "item could not be found"), Self::DataFileExists => (StatusCode::CONFLICT, "item already has data"), Self::DataFileWithoutMetaFile => { + println!("WARN: a data file without meta file exists."); (StatusCode::INTERNAL_SERVER_ERROR, "internal server error") } + Self::ParseTtl => (StatusCode::BAD_REQUEST, "could not parse ttl"), } .into_response() } @@ -65,6 +78,7 @@ pub struct AppState { client_secret: Option, scopes: Vec, logins: Arc>>, + expire: Arc>>, } #[tokio::main] @@ -82,6 +96,12 @@ async fn main() { .map(|x| x.to_owned()) .collect::>(); + let expire = Arc::new(Mutex::new(BTreeMap::new())); + { + let mut expire = expire.lock().await; + load_expire("data", &mut expire).await; + } + let state: AppState = AppState { path: "data".to_string(), application_base, @@ -90,7 +110,11 @@ async fn main() { client_secret, scopes, logins: Arc::new(Mutex::new(HashMap::new())), + expire: expire.clone(), }; + + tokio::spawn(async move { expire_thread("data".to_string(), expire).await }); + let app = Router::new() .route("/", get(openid::handle_login)) .route("/login/:id", get(openid::handle_callback)) @@ -102,8 +126,14 @@ async fn main() { .unwrap(); } +#[derive(Deserialize)] +pub struct PostQuery { + ttl: Option, +} + async fn post_item( Path(id): Path, + Query(params): Query, State(app_state): State, TypedHeader(content_type): TypedHeader, mut stream: BodyStream, @@ -118,6 +148,21 @@ async fn post_item( .await .is_err() { + let expires_at = if let Some(ttl) = params.ttl { + if let Ok(duration) = parse_duration::parse(&ttl) { + Ok(Utc::now().naive_utc() + chrono::Duration::from_std(duration).unwrap()) + } else if let Ok(ttl) = chrono::NaiveDateTime::from_str(&ttl) { + Ok(ttl) + } else { + Err(Error::ParseTtl) + } + } else { + Ok(Utc::now().naive_utc() + chrono::Duration::days(30)) + }?; + { + app_state.expire.lock().await.insert(expires_at, id.clone()); + } + let mut data_file = File::create(&format!("{}/{}.data", app_state.path, &id)) .await .unwrap(); @@ -127,6 +172,7 @@ async fn post_item( } metadata.mimetype = Some(content_type.to_string()); + metadata.ttl = Some(expires_at); metadata.to_file(&app_state.path, &id).await.unwrap(); @@ -215,3 +261,44 @@ fn sanitize_id(id: String) -> String { .filter(|c| c.is_ascii_alphanumeric()) .collect() } + +async fn expire_thread(path: String, expire_dates: Arc>>) { + loop { + { + let now = Utc::now().naive_utc(); + let mut expire_dates = expire_dates.lock().await; + let mut to_delete = vec![]; + for (expire, id) in expire_dates.iter() { + if expire < &now { + println!("removing {id}"); + fs::remove_file(&format!("{}/{}.meta", &path, &id)) + .await + .unwrap(); + fs::remove_file(&format!("{}/{}.data", &path, &id)) + .await + .unwrap(); + to_delete.push(*expire); + } else { + break; + } + } + for expire in to_delete.iter() { + expire_dates.remove(expire); + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} +async fn load_expire(path: &str, expire: &mut BTreeMap) { + let mut files = fs::read_dir(path).await.unwrap(); + while let Some(file) = files.next_entry().await.unwrap() { + let name = file.file_name().into_string().unwrap_or(String::new()); + if name.ends_with(".meta") { + let id = name.replace(".meta", ""); + let metadata = Metadata::from_file(path, &id).await.unwrap(); + if let Some(ttl) = metadata.ttl { + expire.insert(ttl, id); + } + } + } +} diff --git a/src/metadata.rs b/src/metadata.rs index 78f1cf7..3defd0d 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -1,3 +1,4 @@ +use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; use tokio::{ fs::File, @@ -16,6 +17,7 @@ pub enum Error { #[derive(Serialize, Deserialize)] pub struct Metadata { pub subject: String, + pub ttl: Option, pub mimetype: Option, } diff --git a/src/openid.rs b/src/openid.rs index acfb669..17dcc83 100644 --- a/src/openid.rs +++ b/src/openid.rs @@ -192,6 +192,7 @@ pub async fn handle_callback( let metadata = Metadata { subject, mimetype: None, + ttl: None, }; metadata.to_file(&state.path, &id).await.unwrap();