The filesharing service everyone either hates or loves
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
QuadFile/run.py

464 lines
18 KiB

#!/usr/bin/env python3
from datetime import datetime, timedelta
import json
import logging
import os
import random
import re
# import magic
import secrets
import sys
import time
from random import randint
from threading import Thread, Timer
from flask import (Flask, abort, make_response, redirect, render_template,
request, send_from_directory, url_for)
from werkzeug.middleware.proxy_fix import ProxyFix
from werkzeug.utils import secure_filename
# Import our configuration
from conf import config
# Import QuadFile stuff
from QuadFile import application, db
from QuadFile.output import print_log, time_to_string
app = Flask(__name__, template_folder=config['THEME_FOLDER'] + "/templates", static_folder=config['THEME_FOLDER'] + "/static")
app.config['EXPLAIN_TEMPLATE_LOADING'] = False
app.wsgi_app = ProxyFix(app.wsgi_app)
# TODO: Try to turn these into functions or something I dunno
print_log('Main', 'Running in "' + os.getcwd() + '"')
print_log('Main', 'Checking for data folder')
if not os.path.exists(config['UPLOAD_FOLDER']):
print_log('Warning', 'Data folder not found, creating')
os.makedirs(config['UPLOAD_FOLDER'])
if not os.path.exists('files.db'):
print_log('Warning', 'Database not found, attempting to create')
os.system('sqlite3 files.db < schema.sql')
if not os.path.exists('files.db'):
print_log('Warning', 'Could not create database. Is sqlite3 available?')
quit()
else:
print_log('Notice', 'Database created')
if config["METRICS"]:
if not os.path.exists('stats.db'):
print_log('Warning', 'Stats Database not found, attempting to create')
os.system('sqlite3 stats.db < statsschema.sql')
if not os.path.exists('stats.db'):
print_log('Warning', 'Could not create stats database. Is sqlite3 available?')
quit()
else:
print_log('Notice', 'Database created')
if config["EXTENDED_DEBUG"] == False:
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
print_debug = config["DEBUG"]
def cleaner_thread():
# Call itself again after the interval
cleaner = Timer(config["CLEAN_INTERVAL"], cleaner_thread)
cleaner.daemon = True # Daemons will attempt to exit cleanly along with the main process, which we want
cleaner.start()
print_log('Thread', 'Cleaner prepared', print_debug)
# Actual function
delete_old()
def delete_file(file):
print_log('Thread', 'Removing old file "' + file + '"')
if config["METRICS"]:
db.add_stat("timedeletions")
try:
os.remove(os.path.join(config["UPLOAD_FOLDER"], file))
except Exception:
print_log('Warning', 'Failed to delete old file "' + file + '"')
db.delete_entry(file)
def delete_old():
print_log('Thread', 'Cleaner running', print_debug)
#os.chdir(config['UPLOAD_FOLDER'])
maxs = config["MAX_FILESIZE"]
mind = config["MINDAYS"]
maxd = config["MAXDAYS"]
files = [f for f in os.listdir(config['UPLOAD_FOLDER'])]
# TODO: Calculate deletion times at upload, and add to DB (for better stats, and easier handling) #10
if config["USE_0x0_DELETION"]:
for f in files:
stat = os.stat(os.path.join(config['UPLOAD_FOLDER'], f))
systime = time.time()
age = timedelta(seconds = systime - stat.st_mtime).days
maxage = mind + (-maxd + mind) * (stat.st_size / maxs - 1) ** 3
if age >= maxage:
delete_file(f)
else:
targetTime = time.time() - config["TIME"]
old = db.get_old_files(targetTime)
for file in old:
delete_file(file["file"])
def error_page(error, code):
return render_template('error.html', page=config["SITE_DATA"], error=error, code=code)
def allowed_file(filename):
if config["ALLOW_ALL_FILES"]:
print_log('Main', 'All files permitted, no check done', print_debug)
return True
else:
if config["BLACKLIST"]:
if '.' not in filename:
return True
else:
return '.' in filename and filename.rsplit('.', 1)[1] not in config["BANNED_EXTENSIONS"]
else:
return '.' in filename and filename.rsplit('.', 1)[1] in config["ALLOWED_EXTENSIONS"]
def generateMetrics():
# Total QR Scans
metrics = "# QR Code Scanning\n"
metrics = metrics + db.total_qrscan()
# File Stats
metrics = metrics + "# File Stats\n"
# Total Living Files
metrics = metrics + db.total_files_live()
# Total Live Files
metrics = metrics + str(db.total_files())
# Live File Sizes by Type
filenames = db.list_filetypes()
for extension in filenames:
for size in db.list_sizes(extension):
if not size: # Return 0 instead of NoneType
filesize = 0
metrics = metrics + 'filesize_live{type="' + extension + '"} ' + str(filesize) + '\n'
else:
metrics = metrics + 'filesize_live{type="' + extension + '"} ' + str(size) + '\n'
metrics = metrics + "\n"
# Total File Sizes by Type
filenames = db.list_totalfiletypes()
for extension in filenames:
for size in db.list_totalsizes(extension):
if not size: # Return 0 instead of NoneType
filesize = 0
metrics = metrics + 'filesize_total{type="' + extension + '"} ' + str(filesize) + '\n'
else:
metrics = metrics + 'filesize_total{type="' + extension + '"} ' + str(size[0]) + '\n'
metrics = metrics + "\n"
# Live Files by Type
filenames = db.list_filetypes()
for extension in filenames:
for count in db.live_typecount(extension):
metrics = metrics + 'filecount_live{type="' + extension + '"} ' + str(count[0]) + '\n'
metrics = metrics + "\n"
# Total Files by Type
total_filetypes = db.total_filetype()
for filetypes in total_filetypes:
metrics = metrics + 'filecount_total{type="' + filetypes[0] + '"} ' + str(filetypes[1]) + '\n'
metrics = metrics + "\n"
# Total Views per Type
filenames = db.list_totalfiletypes()
for extension in filenames:
for size in db.total_fileview(extension):
metrics = metrics + 'filetype_views_total{type="' + extension + '"} ' + str(size[0]) + '\n'
metrics = metrics + "\n"
# File Deletion
metrics = metrics + "# Longest File Deletion Expiry\n"
metrics = metrics + "total_deletions_url " + str(db.read_stat('urldeletions')) + "\n"
metrics = metrics + "total_deletions_time " + str(db.read_stat('timedeletions')) + "\n"
metrics = metrics + "longest_expiry " + str(db.longest_deletiontime()) + "\n"
metrics = metrics + "# Average File Deletion Expiry\n"
metrics = metrics + "average_expiry " + str(db.average_deletiontime()) + "\n"
metrics = metrics + "\n"
# Page Errors
metrics = metrics + "# Total Page Errors\n"
metrics = metrics + 'errors{code="403"} ' + str(db.read_stat(403)) + "\n"
metrics = metrics + 'errors{code="404"} ' + str(db.read_stat(404)) + "\n"
metrics = metrics + 'errors{code="413"} ' + str(db.read_stat(413)) + "\n"
metrics = metrics + 'errors{code="418"} ' + str(db.read_stat(418)) + "\n"
metrics = metrics + 'errors{code="500"} ' + str(db.read_stat(500)) + "\n"
metrics = metrics + "total_invalid_deletion_key " + str(db.read_stat('invaliddeletionkey')) + "\n\n"
# Page Views
metrics = metrics + "# Total Page Views\n"
metrics = metrics + 'views{page="about"} ' + str(db.read_stat('aboutviews')) + "\n"
metrics = metrics + 'views{page="faq"} ' + str(db.read_stat('faqviews')) + "\n"
metrics = metrics + 'views{page="services"} ' + str(db.read_stat('servicesviews')) + "\n"
metrics = metrics + 'views{page="welcome"} ' + str(db.read_stat('welcomeviews')) + "\n"
metrics = metrics + 'views{page="dmca"} ' + str(db.read_stat('dmcaviews')) + "\n"
metrics = metrics + 'views{page="czb"} ' + str(db.read_stat('czb')) + "\n\n"
return metrics
@app.route('/', methods=['GET', 'POST'])
def upload_file():
if request.method == 'POST':
print_log('Web', 'New file received')
if not application.basicauth(request.headers.get('X-Hyozan-Auth'), config["KEY"]):
abort(403)
data = dict()
file = request.files['file']
# Only continue if a file that's allowed gets submitted.
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
if '.' in filename: #check if filename has a .(to check if it should split ext)
filename = secrets.token_urlsafe(3) + '.' + filename.rsplit('.',1)[1]
else:
filename = secrets.token_urlsafe(3)
# Do Metrics Stuffs
if config["METRICS"]:
# Calculate File Size
file = request.files['file']
file.seek(0, 2)
filesize = file.tell()
file.seek(0, 0) # seek back to start of file so it actually saves
# Strip nasty filetypes for sql injection, then add it for totals
if '.' not in filename:
db.addstats_file("None", filesize)
else:
filetype = filename.rsplit('.',1)[1]
db.addstats_file(filetype, filesize)
else:
filesize = 0
deletekey = secrets.token_urlsafe(10)
thread1 = Thread(target = db.add_file, args = (filename, deletekey, filesize))
thread1.start()
print_log('Thread', 'Adding file to DB', print_debug)
file.save(os.path.join(config['UPLOAD_FOLDER'], filename))
thread1.join()
data["file"] = filename
data["url"] = request.host_url + filename
if config["GEN_DELETEKEY"]:
data["deletionurl"] = request.host_url + "delete/" + deletekey
# Add deletion time to the database - not used yet but once the main server has
# all the files in db with it, then I can use it to delete files properly.
if config["USE_0x0_DELETION"]:
maxs = config["MAX_FILESIZE"]
mind = config["MINDAYS"]
maxd = config["MAXDAYS"]
# Get filesize
stat = os.stat(os.path.join(config['UPLOAD_FOLDER'], filename))
# Calculate how long the file should be alive for
lifetime = mind + (-maxd + mind) * (stat.st_size / maxs - 1) ** 3
# Add max age to current time to get the proper time it should delete
deletiontime = datetime.now() + timedelta(days=lifetime)
db.add_deletiontime(deletiontime, filename)
else:
deletiontime = time.time() + config["TIME"]
deletiontime = datetime.fromtimestamp(deletiontime)
db.add_deletiontime(deletiontime, filename)
print_log('Main', 'New file processed "' + filename + '"')
try:
if request.form["source"] == "web":
print_log('Web', 'Returned link page for "' + filename + '"', print_debug)
return render_template('link.html', data=data, page=config["SITE_DATA"])
except Exception:
print_log('Web', 'No web reported in form, returned JSON', print_debug)
return json.dumps(data)
else:
print_log('Notice', 'Forbidden file received')
return error_page(error="This file isn't allowed, sorry!", code=403)
# Return Web UI if we have a GET request
elif request.method == 'GET':
print_log('Web', 'Hit upload page', print_debug)
return render_template('upload.html', page=config["SITE_DATA"])
# Def all the static pages
@app.route('/about')
def about():
if config["METRICS"]:
db.add_stat("aboutviews")
return render_template('about.html', page=config["SITE_DATA"])
@app.route('/faq')
def faq():
if config["METRICS"]:
db.add_stat("faqviews")
return render_template('faq.html', page=config["SITE_DATA"])
@app.route('/services')
def services():
if config["METRICS"]:
db.add_stat("servicesviews")
return render_template('services.html', page=config["SITE_DATA"])
@app.route('/welcome')
def welcome():
if config["METRICS"]:
db.add_stat("welcomeviews")
video = random.choice(os.listdir(config['THEME_FOLDER'] + "/static/welcome/"))
return render_template('welcome.html', page=config["SITE_DATA"], video=video)
@app.route('/dmca')
def dmca():
if config["METRICS"]:
db.add_stat("dmcaviews")
video = random.choice(os.listdir(config['THEME_FOLDER'] + "/static/dmca/"))
return render_template('dmca.html', page=config["SITE_DATA"], video=video)
@app.route('/czb')
def czb():
if config["METRICS"]:
db.add_stat("czb")
return render_template('czb.html', page=config["SITE_DATA"])
@app.route('/qr')
def qr():
if config["METRICS"]:
db.add_qrscan(request.environ.get('HTTP_X_FORWARDED_FOR', request.remote_addr))
video = random.choice(os.listdir(config['THEME_FOLDER'] + "/static/welcome/"))
return render_template('qr.html', page=config["SITE_DATA"], video=video)
# Static resources that browsers spam for
@app.route('/favicon.ico')
def favicon():
return send_from_directory('static', 'favicon.ico')
@app.route('/apple-touch-icon.png')
def appleTouch():
return send_from_directory('static', 'logo/152px.png')
@app.route('/robots.txt')
def robotsTxt():
return send_from_directory('static', 'robots.txt')
# Custom 404
@app.errorhandler(404)
def page_not_found(e):
db.add_stat("404")
return error_page(error="We couldn't find that. Are you sure you know what you're looking for?", code=404), 404
@app.errorhandler(500)
def internal_error(e):
db.add_stat("500")
return error_page(error="Unknown error, try your upload again, or contact the admin.", code=500), 500
@app.errorhandler(403)
def no_permission(e):
db.add_stat("403")
return error_page(error="Permission denied, no snooping.", code=403), 403
@app.route('/<filename>', methods=['GET'])
def get_file(filename):
print_log('Web', 'Hit "' + filename + '"')
# try:
db.update_file(filename)
if config["METRICS"]:
if '.' not in filename:
db.add_fileview("None")
else:
db.add_fileview(filename.rsplit('.', 1)[1])
# except Exception:
# print_log('Warning', 'Unable to update access time (and view) Is the file in the database?')
if config["X-ACCEL-REDIRECT"]:
r = make_response()
r.headers['Content-Type'] = ''
r.headers['X-Accel-Redirect'] = os.path.join("/" + config["UPLOAD_FOLDER"], filename)
return r
# return text/plain for files with no extensions
if '.' not in filename:
return send_from_directory(config['UPLOAD_FOLDER'], filename, mimetype="text/plain")
# or if they're in the unsafe ext config.
elif filename.endswith(config["UNSAFE_EXTENSIONS"]):
return send_from_directory(config['UPLOAD_FOLDER'], filename, mimetype="text/plain")
# Else, send and let flask guess the mimetype
else:
return send_from_directory(config['UPLOAD_FOLDER'], filename)
@app.route('/delete', methods=['POST'])
def delete_file_key_api():
deletekey = (request.form['key'])
if config["GEN_DELETEKEY"]:
df = db.get_file_from_key(deletekey)
if not df:
print_log('Web', 'Someone used an invalid deletion key', print_debug)
if config["METRICS"]:
db.add_stat("invaliddeletionkey")
return "Error: Invalid deletion key!\n"
for file in df:
print_log('Thread', 'Deleting ' + file["file"] + ' with deletion key')
if config["METRICS"]:
db.add_stat("urldeletions")
try:
os.remove(os.path.join(config["UPLOAD_FOLDER"], file["file"]))
db.delete_entry(file["file"])
except Exception:
print_log('Warning', 'Failed to delete ' + file["file"])
return "Error: Failed to delete file, not exactly sure why though\n"
else:
return 'Deletion keys are not enabled on this server.\n'
return 'File deleted!\n'
@app.route('/delete/<deletekey>', methods=['GET'])
def delete_file_key(deletekey):
if config["GEN_DELETEKEY"]:
df = db.get_file_from_key(deletekey)
if not df:
print_log('Web', 'Someone used an invalid deletion key')
if config["METRICS"]:
db.add_stat("invaliddeletionkey")
return error_page(error="Invalid deletion key, double check your key!", code=403), 403
for file in df:
print_log('Thread', 'Deleting ' + file["file"] + ' with deletion key')
if config["METRICS"]:
db.add_stat("urldeletions")
try:
os.remove(os.path.join(config["UPLOAD_FOLDER"], file["file"]))
db.delete_entry(file["file"])
except Exception:
print_log('Warning', 'Failed to delete ' + file["file"] + ' : Invalid key')
return error_page(error="Failed to delete file, just uhh.. go back to the main page", code=500), 500
else:
return error_page(error="Deletion keys are not enabled on this server.", code=403), 403
return redirect('/')
# Configure nginx to use these urls as custom error pages
@app.route('/error/<int:error>')
def nginx_error(error):
# out here doing 2iq plays because https://www.python.org/dev/peps/pep-0634/ isn't ready.
if error == 413:
return error_page(error="The file you uploaded was too large for the server to handle.", code=413), 413
elif error == 403: # Block IPs with your web server and return /error/403 for this page
return error_page(error="Permission Denied - No snooping allowed thanks :)", code=403), 403
elif error == 404:
return error_page(error="We couldn't find that. Are you sure you know what you're looking for?", code=404), 404
elif error == 418:
return error_page(error="ahaha teapot get it?", code=418), 418
elif error == 500:
return error_page(error="Internal server error, try again or contact the admin", code=500), 500
else:
return error_page(error="We literally have no idea what just happened, try again or contact the admin", code="Unknown")
# Metrics for Grafana
## This should be denied on the proxy, or disabled in config
## Try not to calculate anything on this, use the functions to update a table in
@app.route('/metrics/')
def metrics():
if config["METRICS"]:
response = make_response(generateMetrics(), 200)
response.mimetype = "text/plain"
return response
return error_page(error="Metrics are disabled on this server :(", code=404), 404
if config["DELETE_FILES"]:
cleaner_thread()
print_log('Main', 'Launching app', print_debug)
if __name__ == '__main__':
app.run(
port=config["PORT"],
host=config["HOST"],
debug=config["EXTENDED_DEBUG"]
)