Volkor
/
Ephemeral
1
0
Fork 1
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

finished? metrics and fixed nginx sendfile

main
Volkor 2023-02-22 23:14:28 +11:00
parent c248ecc223
commit 1e540ece8b
Signed by: Volkor
GPG Key ID: BAD7CA8A81CC2DA5
5 changed files with 209 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
docs/info.md
View File

@ -13,13 +13,13 @@ Deleted files should no longer show up in most stats generated, since storing th
### General Stats
- Total number of files alive /right now/ `total_alive`
- Total number of files uploaded `total_dead`
- Total number of files alive /right now/ `total_alive{filetype='*', host="localhost"}`
- Total number of files uploaded `total_alive{filetype='*', host="localhost"}`
- Total number of files uploaded (by x ip) `filecount_ip`
- Total filesize of alive files `filesize_alive`
- Total filesize of dead files `filesize_dead`
- Total bandwidth served for alive files(calculated by views * filesize) `bandwidth_alive`
- Total bandwidth served for dead files(calculated by views * filesize) `bandwidth_dead`
- Total bandwidth served for alive files(calculated by views * filesize) `bandwidth_alive{file="*"}` (this will need to be done per file)
- Total bandwidth served for dead files(calculated by views * filesize) `bandwidth_dead{file="*"}` (this will need to be done per file)
- Geo-map of files uploaded (I'm not too sure how we can do this)
- Geo-map of files served (would require nginx logs)
- Scrape render time `render_time`
@ -31,15 +31,19 @@ Deleted files should no longer show up in most stats generated, since storing th
- Total filesize of alive files `filesize_alive{filetype=mp4}`
- Total filesize of dead files `filesize_dead{filetype=mp4}`
- Total number of views for dead files `views_dead{filetype=mp4}`
- Total number of views for alive files `(count(file{views}))`
- Filesize Graph (Average/total? filesize per filetype)
- Filesize Graph (Filesize vs lifetime)
- Total number of views for alive files `views_alive{filetype=mp4}`
- Total bandwidth for mimetype `file_bandwidth{mimetype="*"}` ()
- Filesize Graph (Average/total? filesize per filetype) ?
- Filesize Graph (Filesize vs lifetime) ?
### File Stats (Pick a individual file from a list, or even multiple?)
- File Stats `file{file=hUMZCp.jpg filesize=2345677 filetype=jpg views=123, expiry=11111111}`
- Total Views on file `file_views{file="bleh.txt"}`
- Total Bandwidth per file `file_bandwidth{file="bleh.txt" mimetype="text/plain"}`
- File Size `filesize{file=hUMZCp.jpg}`
- ? File Mimetype `filetype{file="hUMZCp.jpg"} "text/plain"`
- File Mimetype `filetype{file="hUMZCp.jpg"} "text/plain"`
### Malicious/Error Stats

10
schema.sql
View File

@ -1,6 +1,6 @@
-- Use Write-Ahead-Logging --
-- This speeds up queries by about 2x --
PRAGMA journal_mode=WAL;
-- PRAGMA journal_mode=WAL;
-- PRAGMA synchronous = OFF; -- Disabled by default, can corrupt db if the computer suffers a catastrophic crash (or power failure)
-- This drops the existing files table, and re-creates it. --
@ -26,4 +26,10 @@ CREATE TABLE 'qrscan' (
IP TEXT NOT NULL,
useragent TEXT NOT NULL,
version INTEGER NOT NULL
);
);
-- Dummy file to stop compile error, temporary fix --
-- This is some seriously wacked up formatting.
INSERT INTO files (
file, mimetype, expiry, adminkey, accessed, filesize, ip, domain)
VALUES ( 'dummy','text/plain','1', 'dummyadminkey','0', '0', '127.0.0.1','localhost');

198
src/db.rs
View File

@ -1,5 +1,5 @@
use std::{collections::HashMap, hash::Hash, time::SystemTime};
use sqlx::{query, sqlite::SqliteQueryResult, Pool, Sqlite};
use sqlx::{sqlite::SqliteQueryResult, Pool, Sqlite};
use std::{collections::HashMap, time::SystemTime};
// This struct is used to store values for metrics file stats.
pub struct FileMetric {
@ -10,6 +10,17 @@ pub struct FileMetric {
pub expiry: i64,
}
pub struct MimetypeMetric {
pub mimetype: String,
pub host: String,
pub alive_files: i64,
pub dead_files: i64,
pub filesize_alive: i64,
pub filesize_dead: i64,
pub views_alive: i64,
pub views_dead: i64,
}
// Adding a file to the database
// TODO: Fix panic on fileadd with same filename (even if isDeleted) (UNIQUE constraint)
pub async fn add_file(
@ -24,8 +35,8 @@ pub async fn add_file(
ip: String, // set to the end-user IP of the upload request.
domain: &str, // set to the HOST header of the upload request.
) -> Result<SqliteQueryResult, sqlx::Error> {
let result = sqlx::query!(
"INSERT INTO files (
let result = sqlx::query!(
"INSERT INTO files (
file,
mimetype,
expiry,
@ -35,19 +46,19 @@ pub async fn add_file(
ip,
domain)
VALUES ( ?,?,?,?,?,?,?,? )",
file,
mimetype,
expiry,
adminkey,
accessed,
filesize,
ip,
domain
)
.execute(sqlconn)
.await;
tracing::debug!("add_file.filetype.not_none(Added file to the database.)");
result
file,
mimetype,
expiry,
adminkey,
accessed,
filesize,
ip,
domain
)
.execute(sqlconn)
.await;
tracing::debug!("add_file.filetype.not_none(Added file to the database.)");
result
// Will need to add another else if for expiry_override if added later.
// TODO: Check for row affected, and give a Result
@ -94,7 +105,7 @@ pub async fn check_adminkey(sqlconn: &Pool<Sqlite>, adminkey: String) -> Option<
.fetch_one(sqlconn)
.await;
if result.is_err() {
return None;
None
} else {
let filename: String = result.unwrap().file;
tracing::debug!("check_adminkey(filename: {:?})", filename);
@ -115,7 +126,7 @@ pub async fn get_mimetype(sqlconn: &Pool<Sqlite>, file: String) -> Option<String
.fetch_one(sqlconn)
.await;
if result.is_err() {
return None;
None
} else {
let mimetype: String = result.unwrap().mimetype.unwrap_or("text/plain".to_string());
tracing::debug!("get_mimetype(filename: {:?})", mimetype);
@ -190,7 +201,7 @@ pub async fn get_accesss_time(sqlconn: &Pool<Sqlite>, filename: String) -> i32 {
filename,
accesstime.clone()
);
accesstime as i32
accesstime
}
}
@ -231,7 +242,7 @@ pub async fn get_old_files(sqlconn: &Pool<Sqlite>) -> Vec<String> {
// Updating the expiry_override of a file.
fn update_expiry_override() {
// implement this I guess.
todo!()
}
// Globally important stats.
@ -309,103 +320,11 @@ pub async fn total_dead_filesize(sqlconn: &Pool<Sqlite>) -> Option<u128> {
}
}
// This function queries db for the number of alive files, grouped by filetype.
// Returns a Vec containing the filetype as a String, and the number of alive files.
pub async fn total_alive_filetype(sqlconn: &Pool<Sqlite>) -> Option<HashMap<String, i32>> {
let result = sqlx::query!(
"SELECT COUNT(file) as filecount, mimetype
FROM files
WHERE isdeleted == 0
GROUP BY mimetype",
)
.fetch_all(sqlconn)
.await;
let mut filecount: HashMap<String, i32> = HashMap::new();
if result.is_err() {
// If Error, return none and log.
tracing::error!("Problem getting total alive files by ip: {:?}", result);
None
} else {
for row in result.unwrap() {
filecount.insert(
row.mimetype
.expect("Something went very wrong while getting the alive filetype count."),
row.filecount.try_into().unwrap(),
);
}
Some(filecount)
}
}
// This function queries db for the number of dead files, grouped by filetype.
// Returns a Hashmap containing the filetype as a String, and the number of dead files as i64.
pub async fn total_dead_filetype(sqlconn: &Pool<Sqlite>) -> Option<HashMap<String, i64>> {
let result = sqlx::query!(
"SELECT COUNT(file) as filecount, mimetype
FROM files
WHERE isdeleted == 1
GROUP BY mimetype",
)
.fetch_all(sqlconn)
.await;
let mut filecount: HashMap<String, i64> = HashMap::new();
if result.is_err() {
// If Error, return none and log.
tracing::error!("Problem getting total dead files by ip: {:?}", result);
None
} else {
for row in result.unwrap() {
filecount.insert(
row.mimetype
.expect("Something went very wrong while getting the dead filetype count."),
row.filecount.into(),
);
}
Some(filecount)
}
}
// This fucntion queriest the db for number of dead files, grouped by filetype.
// Returns a hashmap containing a String (filetype) and i32 (views).
pub async fn get_dead_fileviews(sqlconn: &Pool<Sqlite>) -> Option<HashMap<String, i64>> {
let result = sqlx::query!(
"SELECT mimetype, views
FROM files
WHERE isDeleted == 1
GROUP BY mimetype"
)
.fetch_all(sqlconn)
.await;
let mut deadviews: HashMap<String, i64> = HashMap::new();
if result.is_err() {
// If Error, return none and log.
tracing::error!("Problem getting total views of dead files: {:?}", result);
None
} else {
for row in result.unwrap() {
deadviews.insert(
row.mimetype
.expect("Something went very wrong while getting the dead filetype views."),
row.views
.expect("Something went very wrong while getting the dead views"),
);
}
tracing::debug!("deadviews: {:?}", deadviews);
Some(deadviews)
}
}
// This function queries the db for the filesize for /each/ alive file. - We won't need to do this for dead files
// since they were alive at some point, and when they are removed from the alive list, we don't really care.
// This returns a Hashmap of the filename, filetype, and filesize inside a Vector. (How messy)
// We want to group them by file, or filetype, or total in grafana, so we need to label it properly.
pub async fn get_filemetrics(sqlconn: &Pool<Sqlite>) -> Option<Vec<FileMetric>> {
pub async fn get_file_metrics(sqlconn: &Pool<Sqlite>) -> Option<Vec<FileMetric>> {
let result = sqlx::query!(
"SELECT file, filesize, mimetype, views, expiry
FROM files
@ -437,3 +356,54 @@ pub async fn get_filemetrics(sqlconn: &Pool<Sqlite>) -> Option<Vec<FileMetric>>
Some(filevec)
}
}
// This function queries the db for metrics related to mimetypes.
// Returns a Vec of if the MimetypeMetric Struct.
pub async fn get_mimetype_metrics(sqlconn: &Pool<Sqlite>) -> Option<Vec<MimetypeMetric>> {
// Get stats about the files
// This is insane, ChatGPT wrote this entire query. I did no changes at all and it just worked.
// I cannot wait until it takes our jobs.
let result = sqlx::query!(
"SELECT mimetype, domain,
SUM(CASE WHEN isDeleted = 0 THEN views ELSE 0 END) AS alive_views,
SUM(CASE WHEN isDeleted = 1 THEN views ELSE 0 END) AS dead_views,
SUM(CASE WHEN isDeleted = 0 THEN 1 ELSE 0 END) AS alive_files,
SUM(CASE WHEN isDeleted = 1 THEN 1 ELSE 0 END) AS dead_files,
SUM(CASE WHEN isDeleted = 0 THEN filesize ELSE 0 END) AS alive_filesize,
SUM(CASE WHEN isDeleted = 1 THEN filesize ELSE 0 END) AS dead_filesize
FROM files
GROUP BY mimetype, domain;",
)
.fetch_all(sqlconn)
.await;
// Initialise the List of structs
let mut mimevec: Vec<MimetypeMetric> = Vec::new();
if result.is_err() {
tracing::error!("Problem getting mimetype metrics: {:?}", result);
None
} else {
// For each mimetype
for row in result.unwrap() {
// For each row (file), add it to the struct
let mimetype = MimetypeMetric {
mimetype: row.mimetype.unwrap(),
host: row.domain,
alive_files: row.alive_files.unwrap() as i64,
dead_files: row.dead_files.unwrap() as i64,
filesize_alive: row.alive_filesize.unwrap() as i64,
filesize_dead: row.dead_filesize.unwrap() as i64,
views_alive: row.alive_views.unwrap() as i64,
views_dead: row.dead_views.unwrap() as i64,
};
// Then add the struct to the Vec
mimevec.push(mimetype);
}
// For each dead mimetype
// Return vec or return nothing.
Some(mimevec)
}
}

32
src/engine.rs
View File

@ -23,37 +23,15 @@ pub async fn generate_filename(config: Config, filename: String) -> String {
if v.len() == 1 {
// Length is 1, meaning no file extension, so go ahead and generate a 'size' nanoid!
tracing::debug!("generate_filename(v.len == 1, no extension.)");
return nanoid!(size);
nanoid!(size)
} else {
// Take the last 'element', and add that to the end of a nanoid.
tracing::debug!("generate_filename(v.len != 1, re-adding extension.)");
return nanoid!(size) + "." + v.last().unwrap();
}
};
filename
}
// Grab the filetype from the filename - "bleh.filetype" Returns a Some IF there is a filetype.
pub fn get_filetype(filename: &String) -> Option<String> {
// Split the filename into a Vector
let v: Vec<&str> = filename.split('.').collect();
tracing::debug!("get_filetype(v): {:?}", v);
// Check if the Vector has only 1 'element' - means we can ignore extension.
// This works because a file like 'file.bleh' separates into ["file", ".", "bleh"], while 'bleh' separates into ["bleh"]
if v.len() == 1 {
// Length is 1, meaning no file extension, so lets return 'None'
None
} else {
// Length is > 1, meaning there is a file extension. so lets return it. (or there's 0, and this filename is empty somehow)
Some(
v.last()
.expect("filename is empty, you /shouldn't/ ever see this.")
.to_string()
.to_lowercase(),
)
}
}
// Generate a random admin for the uploaded file
// This should check if the adminkey exists for another file, and regenerate if it does.
pub async fn generate_adminkey(sqlconn: &Pool<Sqlite>) -> String {
@ -75,7 +53,7 @@ pub async fn calculate_expiry(sqlconn: &Pool<Sqlite>, filename: String, filesize
// Bruh ChatGPT coming in clutch here.
let expiry = match engine {
1 => engine_1(sqlconn, filename.clone()).await,
2 => engine_2(sqlconn, filename.clone(), filesize).await,
2 => engine_2(sqlconn, filesize).await,
3 => engine_3(sqlconn, filename.clone(), filesize).await,
_ => {
tracing::error!("Unknown engine mode: {}", engine);
@ -99,7 +77,7 @@ async fn engine_1(sqlconn: &Pool<Sqlite>, filename: String) -> i32 {
accesstime + expiry_seconds
}
async fn engine_2(sqlthingy: &Pool<Sqlite>, filename: String, filesize: i32) -> i32 {
async fn engine_2(sqlthingy: &Pool<Sqlite>, filesize: i32) -> i32 {
// Do actual calculation.
// file_expiry_* is in seconds.
// TODO: Set these up from a config file, so we can convert to floats here.
@ -125,7 +103,7 @@ async fn engine_2(sqlthingy: &Pool<Sqlite>, filename: String, filesize: i32) ->
async fn engine_3(sqlthingy: &Pool<Sqlite>, filename: String, filesize: i32) -> i32 {
// Do actual calculation.
3
todo!()
}
pub fn guess_ip(req: &mut Request) -> String {

146
src/main.rs
View File

@ -1,7 +1,7 @@
use once_cell::sync::OnceCell;
use ramhorns::{Content, Ramhorns};
use salvo::fs::NamedFile;
use salvo::hyper::header::{HOST, CONTENT_TYPE};
use salvo::hyper::header::{HOST};
use salvo::prelude::*;
use salvo::serve_static::{StaticDir, StaticFile};
use sqlx::SqlitePool;
@ -98,7 +98,7 @@ async fn serve_file(req: &mut Request, res: &mut Response) {
// Get the mimetype of the file.
let mimetype = db::get_mimetype(sqlconn, filename.clone()).await.unwrap();
for mime in mime_unsafe {
// compare each value with the mimetype.
if mime.clone().into_string().unwrap() == mimetype.clone() {
@ -159,14 +159,13 @@ async fn serve_file(req: &mut Request, res: &mut Response) {
.expect("Couldn't find 'nginx_sendfile' in config. :(");
if nginxsendfile {
// Add the content-type header.
res.add_header("Content-Type", mimetype, true).unwrap();
// Add the header, and we're done.
// X-Accel-Redirect lets nginx serve the file directly, instead of us doing all that hard work.
let xsend = "/files/".to_string() + &filename.to_string();
res.add_header("X-Accel-Redirect", xsend, true).unwrap();
// We don't really need to update the content-type header, since nginx handles that (TODO: Test this lol)
return
} else {
// If nginx sendfile is disabled, we need to render the file directly
let filepath = "files/".to_string() + &filename.to_string();
@ -256,7 +255,12 @@ async fn upload(req: &mut Request, res: &mut Response) {
// Grab the mimetype from the request (fallback to text/plain)
let filepart = file.clone();
let mimetype = filepart.headers().get("content-type").unwrap().to_str().unwrap_or("text/plain");
let mimetype = filepart
.headers()
.get("content-type")
.unwrap()
.to_str()
.unwrap_or("text/plain");
tracing::debug!("upload(mimetype): {:?}", mimetype);
// Check if the filetype is on the 'banned' list
@ -337,7 +341,7 @@ async fn upload(req: &mut Request, res: &mut Response) {
sqlconn,
filename.clone(),
mimetype.to_string(),
expiry.clone(),
expiry,
// expiry_override,
adminkey.clone(),
accessed.as_secs() as i32,
@ -385,7 +389,7 @@ async fn upload(req: &mut Request, res: &mut Response) {
#[handler]
async fn serve_static(req: &mut Request, res: &mut Response) {
let headers = req.headers();
let host = headers[HOST].to_str().unwrap_or_else(|_| "None");
let host = headers[HOST].to_str().unwrap_or("None");
match req.uri().path() {
"/services" => {
tracing::info!("New Request: /services");
@ -469,7 +473,7 @@ async fn serve_static(req: &mut Request, res: &mut Response) {
}
#[handler]
async fn serve_metrics(req: &mut Request, res: &mut Response) {
async fn serve_metrics(res: &mut Response) {
// Lets start timing this:
let start = Instant::now();
@ -478,6 +482,10 @@ async fn serve_metrics(req: &mut Request, res: &mut Response) {
// Setup the massive string of metrics
let mut rendered = String::new();
////
// General Stats
////
// Counting how many files each IP has uploaded.
let mut ipcount = db::get_total_uploads_ip(sqlconn).await;
// Loop through each IP and render it as a new line with a label for each IP
@ -490,6 +498,7 @@ async fn serve_metrics(req: &mut Request, res: &mut Response) {
);
}
// Total number of files alive/dead
rendered = format!(
"{}filesize_alive {}\n",
rendered,
@ -501,57 +510,102 @@ async fn serve_metrics(req: &mut Request, res: &mut Response) {
db::total_dead_filesize(sqlconn).await.unwrap()
);
// // Counting how many alive files have been uploaded per filetype
// let mut afilecount = db::total_alive_filetype(sqlconn).await;
// // Loop through each filetype and render it as a new line with a label for each type
// for afc in afilecount.as_mut().unwrap() {
// rendered = format!(
// "{}total_alive{{filetype={}}} {}\n",
// rendered,
// &afc.0.to_string(),
// &afc.1.to_string()
// );
// }
////
// Filetype Stats
////
// Counting how many dead files have been uploaded per filetype
let mut dfilecount = db::total_dead_filetype(sqlconn).await;
// Loop through each filetype and render it as a new line with a label for each type
for dfc in dfilecount.as_mut().unwrap() {
let mimetype_metrics = db::get_mimetype_metrics(sqlconn).await;
// Add a newline here so we can keep it pretty.
rendered = format!("{}\n", rendered);
for row in mimetype_metrics.unwrap() {
// Alive filescount
rendered = format!(
"{}total_dead{{filetype=\"{}\"}} {}\n",
rendered,
&dfc.0.to_string(),
&dfc.1.to_string()
"{}total_alive{{host=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, row.host, row.mimetype, row.alive_files,
);
// Dead filescount
rendered = format!(
"{}total_dead{{host=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, row.host, row.mimetype, row.dead_files,
);
// Alive Filesize
rendered = format!(
"{}filesize_alive{{host=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, row.host, row.mimetype, row.filesize_alive,
);
// Dead Filesize
rendered = format!(
"{}filesize_dead{{host=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, row.host, row.mimetype, row.filesize_dead,
);
// Alive Filesize
rendered = format!(
"{}views_alive{{host=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, row.host, row.mimetype, row.views_alive,
);
// Dead Filesize
rendered = format!(
"{}views_dead{{host=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, row.host, row.mimetype, row.views_dead,
);
// Add an extea newline to split between mimetypes and hosts
rendered = format!("{}\n", rendered);
}
////
// Individual Stats
////
// This is a pain, we're grabbing the individual file stats and parsing them for each file.
let filevec = db::get_filemetrics(sqlconn).await;
let filevec = db::get_file_metrics(sqlconn).await;
// Add a newline here so we can keep it pretty.
rendered = format!("{}\n", rendered);
// For each line in the Vec.
for file in filevec.unwrap() {
// Add the file to the rendered String :)
// View count per file
rendered = format!(
"{}file_expiry{{filename=\"{}\", filesize=\"{}\", filetype=\"{}\", views=\"{}\"}} {}\n",
rendered, file.filename, file.filesize, file.mimetype, file.views, file.expiry,
"{}file_views{{filename=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, file.filename, file.mimetype, file.views,
);
// Size per file
rendered = format!(
"{}file_size{{filename=\"{}\", mimetype=\"{}\"}} {}\n",
rendered, file.filename, file.mimetype, file.filesize,
);
// Bandwidth per file
// Views * filesize
let bandwidth = file.views * file.filesize;
rendered = format!(
"{}file_bandwidth{{filename=\"{}\", mimetype=\"{}\"}} {}\n\n",
rendered, file.filename, file.mimetype, bandwidth,
);
}
// Getting the number of views for all dead filetypes.
let mut deadfileview = db::get_dead_fileviews(sqlconn).await;
// Add a newline here so we can keep it pretty.
rendered = format!("{}\n", rendered);
// Loop through each filetype and render it as a new line with a label for each type
for dfv in deadfileview.as_mut().unwrap() {
rendered = format!(
"{}views_dead{{views=\"{}\"}} {}\n",
rendered,
&dfv.0.to_string(),
&dfv.1.to_string()
);
}
//
////
// Error Stats
////
// Number of files blocked per filetype
// Number of files blocked by IP?
// Number of backend errors (by error type) (plus collision errors)
////
// QR Scanning Stats
////
// TODO
// Add how long it took to get all of those metrics to the page!
let end = Instant::now();
rendered = format!(