Volkor
/
Ephemeral
1
0
Fork 1
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

config, blocked files working, unsafe files broken

main
Volkor 2023-01-20 04:16:05 +11:00
parent 698b99c486
commit a27f60acbf
Signed by: Volkor
GPG Key ID: BAD7CA8A81CC2DA5
7 changed files with 282 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/launch.json vendored
View File

@ -40,7 +40,7 @@
},
"args": [],
"cwd": "${workspaceFolder}",
"env": {"EPHEMERAL_LOG": "debug"}
"env": {"LOG": "debug"}
},
{
"type": "lldb",

140
Cargo.lock generated
View File

@ -220,6 +220,25 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "config"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7"
dependencies = [
"async-trait",
"json5",
"lazy_static",
"nom",
"pathdiff",
"ron",
"rust-ini",
"serde",
"serde_json",
"toml",
"yaml-rust",
]
[[package]]
name = "cookie"
version = "0.16.1"
@ -408,6 +427,12 @@ dependencies = [
"crypto-common",
]
[[package]]
name = "dlv-list"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257"
[[package]]
name = "dotenvy"
version = "0.15.6"
@ -454,6 +479,8 @@ name = "ephemeral"
version = "0.1.0"
dependencies = [
"chrono",
"config",
"lazy_static",
"nanoid",
"once_cell",
"ramhorns",
@ -895,6 +922,17 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json5"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1"
dependencies = [
"pest",
"pest_derive",
"serde",
]
[[package]]
name = "jsonwebtoken"
version = "8.2.0"
@ -941,6 +979,12 @@ dependencies = [
"cc",
]
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "lock_api"
version = "0.4.9"
@ -1149,6 +1193,16 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "ordered-multimap"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
dependencies = [
"dlv-list",
"hashbrown",
]
[[package]]
name = "overload"
version = "0.1.1"
@ -1215,6 +1269,12 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42"
[[package]]
name = "pathdiff"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
[[package]]
name = "pem"
version = "1.1.0"
@ -1230,6 +1290,50 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]]
name = "pest"
version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a"
dependencies = [
"thiserror",
"ucd-trie",
]
[[package]]
name = "pest_derive"
version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "241cda393b0cdd65e62e07e12454f1f25d57017dcc514b1514cd3c4645e3a0a6"
dependencies = [
"pest",
"pest_generator",
]
[[package]]
name = "pest_generator"
version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46b53634d8c8196302953c74d5352f33d0c512a9499bd2ce468fc9f4128fa27c"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pest_meta"
version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef4f1332a8d4678b41966bb4cc1d0676880e84183a1ecc3f4b69f03e99c7a51"
dependencies = [
"once_cell",
"pest",
"sha2",
]
[[package]]
name = "pin-project"
version = "1.0.12"
@ -1495,6 +1599,17 @@ dependencies = [
"winapi",
]
[[package]]
name = "ron"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a"
dependencies = [
"base64 0.13.1",
"bitflags",
"serde",
]
[[package]]
name = "rust-embed"
version = "6.4.2"
@ -1529,6 +1644,16 @@ dependencies = [
"walkdir",
]
[[package]]
name = "rust-ini"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
dependencies = [
"cfg-if",
"ordered-multimap",
]
[[package]]
name = "rustls"
version = "0.20.7"
@ -2314,6 +2439,12 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "ucd-trie"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
[[package]]
name = "unicase"
version = "2.6.0"
@ -2661,3 +2792,12 @@ name = "xxhash-rust"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70"
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]

2
Cargo.toml
View File

@ -20,3 +20,5 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
nanoid = "0.4.0"
chrono = "0.4.23"
rand = "0.8.5"
config = "0.13.3"
lazy_static = "1.4.0"

18
README.md
View File

@ -27,6 +27,24 @@ Eventually we'll have 2 different environments, development and production, each
For now, just run `cargo run` (or `cargo watch -x run` if you're cool.)
### Configuration Options
Configuration is done by settings environment variables in the launch command.
(This definitely isn't because I cannot be bothered implementing a proper config file in code.)
| Variable | Default Value | |
|------------------|---------------|----------------------------------------------------------------------------------|
| HOST | 0.0.0.0 | What IP the application listens on |
| PORT | 8282 | What port the application listens on. |
| EPHEMERAL_LOG | | (Required) Sets the log level output |
| ENGINE | 2 | Sets the engine mode |
| CLEANER_INTERVAL | 1800 | How long the cleaner task runs, in seconds. |
| FILE_EXPIRY_MIN | 7 | (Depends on engine setting) The Minimum a file /should/ exist on the server for. |
| FILE_EXPIRY_MAX | 365 | (Depends on engine setting) The Longest a file /should/ exist on the server for. |
| MAX_FILESIZE* | 1073741824 | The 'cap' for calculating expiry. |
\*MAX_FILESIZE doesn't actually set the maximum allowed filesize, it's only used for calculating the expiry.
## Wierd Security things you should probably be aware of
If you're running outside of a reverse proxy, it's possible to access any file on the file system that is readable by the user.

38
config.toml
View File

@ -2,33 +2,20 @@
host = "0.0.0.0"
port = 8282
# Disables sending files from the backend, and use NGINX to send files directly.
# Honestly, I have no idea if this is more efficient than sending direct, but the old python had it, so this will too.
X-Accel-Redirect = false
[logging]
# Available logging levels
# info - only show critical errors, and usage information
# debug - shows super fancy salvo logging plus above.
level = "info"
[database]
# useless for now, only sqlite is supported.
type = "sqlite"
[database.sqlite]
database_path = "ephemeral.db"
[database.postgres]
database_url = "idk something"
[database.mysql]
database_url = "idk something"
# Enables/Disables metrics generation
metrics = true
[operations]
# This changes how long the random filename should be.
# Leaving it at 6 is good enough for most, since it'll /try/ to regenerate in the case of a collision.
filename_size = "6"
# 1 - time based expiration - last view
# 2 - size based expiration - larger files expire faster
# 3 - aggregate score - large files viewed more often last longer than less popular files.
@ -37,10 +24,19 @@ engine_mode = 1
# How often should the system check for old files.
# Should probably keep this as default. (especially for massive instances)
cleaner_interval = "60"
cleaner_interval = "1800"
# The longest and shortest time a file can be 'alive' on the server.
file_expiry_max = 365
file_expiry_min = 7
# size in bytes - default: 1 Gibibyte
filesize_max = 1073741824
filesize_max = 1073741824
# Files that are not allowed to be uploaded.
# Leave empty to allow all files, and please enter these in lowercase.
banned_extensions = ["exe", "msi", "scr", "com", "cmd"]
# Files that should only ever be rendered as plaintext.
# I REALLY recommend not removing these.
unsafe_extensions = ["htm", "html", "js", "mjs", "css", "log", "php"]

29
src/engine.rs
View File

@ -1,23 +1,18 @@
use config::Config;
////////
use nanoid::nanoid;
use salvo::Request;
use sqlx::{Pool, Sqlite, SqlitePool};
use sqlx::{Pool, Sqlite};
use std::{fs, time::SystemTime};
/// This is just for the ugly functions that we don't really need in the main.rs file.
/// I don't really know what I'm doing, so this makes it looks like im a better developer than I actually am.
////////
use tokio::{
sync::OnceCell,
task,
time::{self, Duration, Interval},
};
use crate::db;
// Generate a random name for the uploaded file.
pub async fn generate_filename(filename: String) -> String {
let size = 6;
// TODO: Get size from config
pub async fn generate_filename(config: Config, filename: String) -> String {
let size = config.get_int("operations.filename_size").expect("Couldn't find 'filename_size' in config. :(") as usize;
// Split the filename into a Vector
let v: Vec<&str> = filename.split('.').collect();
tracing::debug!("generate_filename(v): {:?}", v);
@ -36,7 +31,7 @@ pub async fn generate_filename(filename: String) -> String {
filename
}
// Grab the filetype from the filename - "bleh.filetype" Returns a result IF there is a filetype.
// Grab the filetype from the filename - "bleh.filetype" Returns a Some IF there is a filetype.
pub fn get_filetype(filename: String) -> Option<String> {
// Split the filename into a Vector
let v: Vec<&str> = filename.split('.').collect();
@ -202,11 +197,15 @@ pub fn guess_ip(req: &mut Request) -> String {
"0.0.0.0".to_string()
}
pub async fn delete_file(filename: String) -> std::io::Result<()> {
pub async fn delete_file(file: String) {
// Find the file in the directory, and delete it!
// Set Destination
let dest = format!("files/{}", filename);
fs::remove_file(&dest)?;
tracing::info!("Deleted file: {:?}", dest);
Ok(())
let dest = format!("files/{}", file);
let r = fs::remove_file(&dest);
if r.is_err() {
tracing::error!("Failed to delete file: {:?}", dest);
} else {
tracing::info!("Deleted file: {:?}", dest);
return r.unwrap();
}
}

115
src/main.rs
View File

@ -5,7 +5,7 @@ use salvo::prelude::*;
use salvo::serve_static::{StaticDir, StaticFile};
use sqlx::SqlitePool;
use chrono::{DateTime, TimeZone, Utc};
use chrono::{TimeZone, Utc};
use rand::Rng;
use std::fs::create_dir_all;
use std::path::Path;
@ -15,12 +15,23 @@ use tokio::{task, time};
use tracing_subscriber::filter::EnvFilter;
use tracing_subscriber::fmt;
use tracing_subscriber::prelude::*;
use config::Config;
use lazy_static::lazy_static;
// Import sub-modules.
mod db;
mod engine;
// Setup the global sqlite db
static SQLITE: OnceCell<SqlitePool> = OnceCell::new();
// Setup the config globally because I can't figure out how to pass it to functions I don't call directly.
// This is evaluated at runtime, and not compilation. \o/
lazy_static! {
pub static ref CONFIG: Config = Config::builder()
.add_source(config::File::with_name("config.toml"))
.build()
.unwrap();
}
// This is needed for templating, all the 'variables' go here!
#[derive(Content)]
@ -37,10 +48,9 @@ async fn index(req: &mut Request, res: &mut Response) {
// Get the headers (for Host header stuff thats needed later)
let headers = req.headers();
// build the path we need for the template.
let template_with_host = "./templates/".to_owned() + headers[HOST].to_str().unwrap();
let template_with_host = "./templates/".to_owned() + headers[HOST].to_str().unwrap_or_else(|_| "localhost:8282");
// Now we need to setup the templating engine.
// TODO: replace unwrap with error handling for templates not being found.
let tpls: Ramhorns = Ramhorns::from_folder(template_with_host).unwrap();
let tpls: Ramhorns = Ramhorns::from_folder(template_with_host).expect("Unable to find template, please place the templates correctly!");
let rendered = tpls.get("upload.html").unwrap().render(&"");
// Removed templating for debugging multiple template dirs - I should probably add it back in.
res.render(Text::Html(rendered));
@ -51,7 +61,8 @@ async fn serve_file(req: &mut Request, res: &mut Response) {
let headers = req.headers();
let sqlconn = SQLITE.get().unwrap();
// Check if the filename exists in the DB
let filename: String = req.param("file").unwrap_or_default();
let filename: String = req.param("file").unwrap();
let filetype: String = engine::get_filetype(filename.clone()).unwrap_or("".to_string());
let valid = db::check_filename(sqlconn, filename.clone()).await;
if !valid {
@ -74,10 +85,42 @@ async fn serve_file(req: &mut Request, res: &mut Response) {
return;
}
// override the mimetype if it's part of unsafe extensions
let r#unsafe = CONFIG.get_array("operations.unsafe_extensions").expect("Couldn't find 'unsafe_extensions' in config. :(");
for ext in r#unsafe {
// compare each value with the filetype.
if ext.clone().into_string().unwrap() == filetype.clone() {
tracing::info!("Unsafe Extension Filtered: {:?}", ext.clone().into_string().unwrap());
// Try overriding the content-type, otherwise throw an error.
let addheader = res.add_header("Content-Type", "text/plain", true);
if addheader.is_err() {
tracing::error!("Failed overwriting Content-Type {:?}", ext.clone().into_string().unwrap());
let template_with_host = "./templates/".to_owned() + headers[HOST].to_str().unwrap();
// Now we need to setup the templating engine.
// Yuck we need to setup a struct
let tpls: Ramhorns = Ramhorns::from_folder(template_with_host).unwrap();
let template = TemplateStruct {
domain: String::from(headers[HOST].to_str().unwrap()),
filename: String::from(""),
adminkey: String::from(""),
message1: String::from("Error 500: Internal Server Error"),
message2: String::from(
"This shouldn't happen, but it did. Tell the admin there was a problem overriding the content-type header.",
),
};
let rendered = tpls.get("error.html").unwrap().render(&template);
res.set_status_code(StatusCode::INTERNAL_SERVER_ERROR);
res.render(Text::Html(rendered));
}
};
}
// X-Accel-Redirect lets nginx serve the file directly, instead of us doing all that hard work.
let xsend = "/files/".to_string() + &filename.to_string();
res.add_header("X-Accel-Redirect", xsend, true).unwrap();
// Go through all the headers and print them out, just to check for now!
tracing::debug!("response headers: {:?}", res.headers());
// Get the current unix time
let accessed = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
@ -86,10 +129,14 @@ async fn serve_file(req: &mut Request, res: &mut Response) {
tracing::info!("New File View: {:?}", &filename.to_string());
db::update_fileview(sqlconn, filename.clone(), accessed).await;
// Recalculate expiry for some enginemodes
// Recalculate expiry for enginemode 1
// we don't need filesize here, so it's 0.
let filesize = 0;
// TODO: This will recalculate no matter what, even if engine mode is 2 :/
engine::calculate_expiry(sqlconn, filename.clone(), filesize).await;
// TODO: Add actual file serving from the disk HERE, since salvo's built-in way breaks content-type header.
}
// This takes the adminkey in, and deletes the file that matches it in the DB.
@ -97,7 +144,7 @@ async fn serve_file(req: &mut Request, res: &mut Response) {
async fn delete_file(req: &mut Request, res: &mut Response) {
let headers = req.headers();
let sqlconn = SQLITE.get().unwrap();
let adminkey: &str = req.param("adminkey").unwrap_or_default();
let adminkey: &str = req.param("adminkey").unwrap();
tracing::debug!("delete_file(adminkey): {:?}", adminkey);
// Checks if the adminkey is valid, and the file is active.
let filename = db::check_adminkey(sqlconn, adminkey.to_string()).await;
@ -149,10 +196,32 @@ async fn upload(req: &mut Request, res: &mut Response) {
// Generate new filename.
// TODO: Do all the checks to make sure we actually want to generate a new filename (needs config working)
// Convert Option<&str> to Option<String> (and then generate a new filename for it)
let filename = engine::generate_filename(file.name().unwrap_or("file").to_string()).await;
// We should now check if the filename isn't already in the DB.
let filename = engine::generate_filename(CONFIG.clone(), file.name().unwrap_or("file").to_string()).await;
// TODO: We should now check if the filename isn't already in the DB.
// Grab the filetype from the filename
let filetype = engine::get_filetype(file.name().unwrap_or("file").to_string());
// Check if the filetype is on the 'banned' list
let banned = CONFIG.get_array("operations.banned_extensions").expect("Couldn't find 'banned_extensions' in config. :(");
for ext in banned {
// compare each value with the filetype.
if ext.clone().into_string().unwrap() == filetype.clone().unwrap() {
tracing::info!("Upload was blocked due to blocked extension: {:?}", ext.clone().into_string());
let template_with_host = "./templates/".to_owned() + headers[HOST].to_str().unwrap();
let tpls: Ramhorns = Ramhorns::from_folder(template_with_host).unwrap();
let template = TemplateStruct {
domain: String::from(headers[HOST].to_str().unwrap()),
filename: String::from(""),
adminkey: String::from(""),
message1: String::from("Error 403"),
message2: String::from("That filetype is not allowed."),
};
let rendered = tpls.get("error.html").unwrap().render(&template);
res.set_status_code(StatusCode::FORBIDDEN);
res.render(Text::Html(rendered));
}
}
let adminkey = engine::generate_adminkey(sqlconn).await;
tracing::debug!("upload(filename, adminkey): {:?}, {:?}", filename, adminkey);
@ -252,11 +321,11 @@ async fn upload(req: &mut Request, res: &mut Response) {
#[handler]
async fn serve_static(req: &mut Request, res: &mut Response) {
let headers = req.headers().clone();
let host = headers[HOST].to_str().unwrap();
let host = headers[HOST].to_str().unwrap_or_else(|_| "None");
match req.uri().path() {
"/services" => {
tracing::info!("New Request: /services");
let template_with_host = "./templates/".to_owned() + headers[HOST].to_str().unwrap();
let template_with_host = "./templates/".to_owned() + host;
let tpls: Ramhorns = Ramhorns::from_folder(template_with_host).unwrap();
let template = TemplateStruct {
domain: String::from(headers[HOST].to_str().unwrap()),
@ -342,8 +411,8 @@ async fn main() {
// TODO: Figure out how to make it default to INFO level.
// TODO: Disable salvo_extra::logging for info, add it on debug level instead.
tracing_subscriber::registry()
.with(fmt::layer())
.with(EnvFilter::from_env("EPHEMERAL_LOG"))
.with(fmt::layer().compact()) // Make sure the logging is pretty.
.with(EnvFilter::from_env("LOG")) // Grab the info from the envvar
.init();
// Set up DB Pool!
@ -351,11 +420,10 @@ async fn main() {
// Sets the db pool to the static thingy, so we can access it /anywhere!/
SQLITE.set(pool).unwrap();
// Setup the cleaner thread!
// Get the engine mode from the config
let interval = 1800; // 30 Minutes
// Will awaiting on this wait until the loop is finished? I hope not....
cleaner_thread(interval);
// Initialise the cleaner task
let interval = CONFIG.get_int("operations.cleaner_interval").expect("Couldn't find 'cleaner_interval' in config. :(");
tracing::info!("interval: {}", interval);
cleaner_thread(interval.try_into().expect("Cleaner interval was too long to fit in a i32.... wow"));
// Create the tables if they don't already exist
let (filesdb, qrscandb) = tokio::join!(
@ -398,8 +466,8 @@ async fn main() {
);
// Read environment variables for host and port
let host = env::var("HOST").unwrap_or_else(|_| "127.0.0.1".to_owned());
let port = env::var("PORT").unwrap_or_else(|_| "8282".to_owned());
let host = CONFIG.get_string("server.host").expect("Couldn't find 'host' in config. :(");
let port = CONFIG.get_int("server.port").expect("Couldn't find 'port' in config. :(");
let server_url = format!("{}:{}", host, port);
tracing::info!("Listening on http://{}", server_url);
Server::new(TcpListener::bind(&server_url))
@ -410,7 +478,7 @@ async fn main() {
// This spawns a tokio task to run a interval timer forever.
// the interval timer runs every 'period' seconds.
pub fn cleaner_thread(period: i32) {
let forever = task::spawn(async move {
let _forever = task::spawn(async move {
let mut interval = time::interval(Duration::from_secs(period.try_into().unwrap()));
let sqlconn = SQLITE.get().unwrap();
loop {
@ -422,10 +490,7 @@ pub fn cleaner_thread(period: i32) {
// For each file in old_files, delete them.
for file in old_files {
db::delete_file(sqlconn, file.clone()).await;
let files = engine::delete_file(file.clone()).await;
if files.is_err() {
tracing::info!("Failed to delete file: {:?}", file);
}
engine::delete_file(file.clone()).await;
}
tracing::info!("Cleaner finished")
}