QuadFile/run.py

#!/usr/bin/env python3
from datetime import datetime, timedelta
import json
import logging
import os
import random
import re
# import magic
import secrets
import sys
import time
from random import randint
from threading import Thread, Timer

from flask import (Flask, abort, make_response, redirect, render_template,
                   request, send_from_directory, url_for)
from werkzeug.middleware.proxy_fix import ProxyFix
from werkzeug.utils import secure_filename

# Import our configuration
from conf import config
# Import QuadFile stuff
from QuadFile import application, db
from QuadFile.output import print_log, time_to_string

app = Flask(__name__, template_folder=config['THEME_FOLDER'] + "/templates", static_folder=config['THEME_FOLDER'] + "/static")
app.config['EXPLAIN_TEMPLATE_LOADING'] = False
app.wsgi_app = ProxyFix(app.wsgi_app)

# TODO: Try to turn these into functions or something I dunno
print_log('Main', 'Running in "' + os.getcwd() + '"')
print_log('Main', 'Checking for data folder')
if not os.path.exists(config['UPLOAD_FOLDER']):
  print_log('Warning', 'Data folder not found, creating')
  os.makedirs(config['UPLOAD_FOLDER'])
if not os.path.exists('files.db'):
  print_log('Warning', 'Database not found, attempting to create')
  os.system('sqlite3 files.db < schema.sql')
  if not os.path.exists('files.db'):
    print_log('Warning', 'Could not create database. Is sqlite3 available?')
    quit()
  else:
    print_log('Notice', 'Database created')
if config["METRICS"]:
  if not os.path.exists('stats.db'):
    print_log('Warning', 'Stats Database not found, attempting to create')
    os.system('sqlite3 stats.db < statsschema.sql')
    if not os.path.exists('stats.db'):
      print_log('Warning', 'Could not create stats database. Is sqlite3 available?')
      quit()
    else:
      print_log('Notice', 'Database created')
if config["EXTENDED_DEBUG"] == False:
  log = logging.getLogger('werkzeug')
  log.setLevel(logging.ERROR)

print_debug = config["DEBUG"]

def cleaner_thread():
  # Call itself again after the interval
  cleaner = Timer(config["CLEAN_INTERVAL"], cleaner_thread)
  cleaner.daemon = True # Daemons will attempt to exit cleanly along with the main process, which we want
  cleaner.start()
  print_log('Thread', 'Cleaner prepared', print_debug)
  # Actual function
  delete_old()

def delete_file(file):
  print_log('Thread', 'Removing old file "' + file + '"')
  if config["METRICS"]:
    db.add_stat("timedeletions")
  try:
    os.remove(os.path.join(config["UPLOAD_FOLDER"], file))
  except Exception:
    print_log('Warning', 'Failed to delete old file "' + file + '"')
  db.delete_entry(file)

def delete_old():
  print_log('Thread', 'Cleaner running', print_debug)
  #os.chdir(config['UPLOAD_FOLDER'])
  maxs = config["MAX_FILESIZE"]
  mind = config["MINDAYS"]
  maxd = config["MAXDAYS"]

  files = [f for f in os.listdir(config['UPLOAD_FOLDER'])]
  # TODO: Calculate deletion times at upload, and add to DB (for better stats, and easier handling) #10
  if config["USE_0x0_DELETION"]:
    for f in files:
      stat = os.stat(os.path.join(config['UPLOAD_FOLDER'], f))
      systime = time.time()
      age = timedelta(seconds = systime - stat.st_mtime).days
      maxage = mind + (-maxd + mind) * (stat.st_size / maxs - 1) ** 3
      if age >= maxage:
        delete_file(f)
  else:
    targetTime = time.time() - config["TIME"]
    old = db.get_old_files(targetTime)
    for file in old:
      delete_file(file["file"])

def error_page(error, code):
  return render_template('error.html', page=config["SITE_DATA"], error=error, code=code)


def allowed_file(filename):
  if config["ALLOW_ALL_FILES"]:
    print_log('Main', 'All files permitted, no check done', print_debug)
    return True
  else:
    if config["BLACKLIST"]:
      if '.' not in filename:
        return True
      else:
        return '.' in filename and filename.rsplit('.', 1)[1] not in config["BANNED_EXTENSIONS"]
    else:
      return '.' in filename and filename.rsplit('.', 1)[1] in config["ALLOWED_EXTENSIONS"]

def generateMetrics():
  # Total QR Scans
  metrics = "# QR Code Scanning\n"
  metrics = metrics + db.total_qrscan()
  
  # File Stats
  metrics = metrics + "# File Stats\n"
  
  # Total Living Files
  metrics = metrics + db.total_files_live()
  # Total Live Files
  metrics = metrics + str(db.total_files())
  
  # Live File Sizes by Type
  filenames = db.list_filetypes()
  for extension in filenames:
    for size in db.list_sizes(extension):
      if not size: # Return 0 instead of NoneType
        filesize = 0
        metrics = metrics + 'filesize_live{type="' + extension + '"} ' + str(filesize) + '\n'
      else:
        metrics = metrics + 'filesize_live{type="' + extension + '"} ' + str(size) + '\n'
  metrics = metrics + "\n"
  
  # Total File Sizes by Type
  filenames = db.list_totalfiletypes()
  for extension in filenames:
    for size in db.list_totalsizes(extension):
      if not size: # Return 0 instead of NoneType
        filesize = 0
        metrics = metrics + 'filesize_total{type="' + extension + '"} ' + str(filesize) + '\n'
      else:
        metrics = metrics + 'filesize_total{type="' + extension + '"} ' + str(size[0]) + '\n'
  metrics = metrics + "\n"
  
  # Live Files by Type
  filenames = db.list_filetypes()
  for extension in filenames:
    for count in db.live_typecount(extension):
      metrics = metrics + 'filecount_live{type="' + extension + '"} ' + str(count[0]) + '\n'
  metrics = metrics + "\n"
  
  # Total Files by Type
  total_filetypes = db.total_filetype()
  for filetypes in total_filetypes:
    metrics = metrics + 'filecount_total{type="' + filetypes[0] + '"} ' + str(filetypes[1]) + '\n'
  metrics = metrics + "\n"

  # Total Views per Type
  filenames = db.list_totalfiletypes()
  for extension in filenames:
    for size in db.total_fileview(extension):
      metrics = metrics + 'filetype_views_total{type="' + extension + '"} ' + str(size[0]) + '\n'
  metrics = metrics + "\n"
  
  # File Deletion
  metrics = metrics + "# Longest File Deletion Expiry\n"
  metrics = metrics + "total_deletions_url " + str(db.read_stat('urldeletions')) + "\n"
  metrics = metrics + "total_deletions_time " + str(db.read_stat('timedeletions')) + "\n"
  metrics = metrics + "longest_expiry " + str(db.longest_deletiontime()) + "\n"
  
  metrics = metrics + "# Average File Deletion Expiry\n"
  metrics = metrics + "average_expiry " + str(db.average_deletiontime()) + "\n"
  metrics = metrics + "\n"
  
  # Page Errors
  metrics = metrics + "# Total Page Errors\n"
  metrics = metrics + 'errors{code="403"} ' + str(db.read_stat(403)) + "\n"
  metrics = metrics + 'errors{code="404"} ' + str(db.read_stat(404)) + "\n"
  metrics = metrics + 'errors{code="413"} ' + str(db.read_stat(413)) + "\n"
  metrics = metrics + 'errors{code="418"} ' + str(db.read_stat(418)) + "\n"
  metrics = metrics + 'errors{code="500"} ' + str(db.read_stat(500)) + "\n"
  metrics = metrics + "total_invalid_deletion_key " + str(db.read_stat('invaliddeletionkey')) + "\n\n"
  
  # Page Views
  metrics = metrics + "# Total Page Views\n"
  metrics = metrics + 'views{page="about"} ' + str(db.read_stat('aboutviews')) + "\n"
  metrics = metrics + 'views{page="faq"} ' + str(db.read_stat('faqviews')) + "\n"
  metrics = metrics + 'views{page="services"} ' + str(db.read_stat('servicesviews')) + "\n"
  metrics = metrics + 'views{page="welcome"} ' + str(db.read_stat('welcomeviews')) + "\n"
  metrics = metrics + 'views{page="dmca"} ' + str(db.read_stat('dmcaviews')) + "\n"
  metrics = metrics + 'views{page="czb"} ' + str(db.read_stat('czb')) + "\n\n"  
  
  return metrics

@app.route('/', methods=['GET', 'POST'])
def upload_file():
  if request.method == 'POST':
    print_log('Web', 'New file received')
    if not application.basicauth(request.headers.get('X-Hyozan-Auth'), config["KEY"]):
      abort(403)
    data = dict()
    file = request.files['file']

    # Only continue if a file that's allowed gets submitted.
    if file and allowed_file(file.filename):
      filename = secure_filename(file.filename)
      if '.' in filename: #check if filename has a .(to check if it should split ext)
        filename = secrets.token_urlsafe(3) + '.' + filename.rsplit('.',1)[1]
      else:
        filename = secrets.token_urlsafe(3)
      
      # Do Metrics Stuffs
      if config["METRICS"]:
        # Calculate File Size
        file = request.files['file']
        file.seek(0, 2)
        filesize = file.tell()
        file.seek(0, 0) # seek back to start of file so it actually saves
        
        # Strip nasty filetypes for sql injection, then add it for totals
        
        if '.' not in filename:
          db.addstats_file("None", filesize)
        else:
          filetype = filename.rsplit('.',1)[1]
          db.addstats_file(filetype, filesize)
      else:
        filesize = 0

      deletekey = secrets.token_urlsafe(10)
      thread1 = Thread(target = db.add_file, args = (filename, deletekey, filesize))
      thread1.start()
      print_log('Thread', 'Adding file to DB', print_debug)
      file.save(os.path.join(config['UPLOAD_FOLDER'], filename))
      thread1.join()

      data["file"] = filename
      data["url"] = request.host_url + filename
      if config["GEN_DELETEKEY"]:
        data["deletionurl"] = request.host_url + "delete/" + deletekey

      # Add deletion time to the database - not used yet but once the main server has
      # all the files in db with it, then I can use it to delete files properly.
      if config["USE_0x0_DELETION"]:
        maxs = config["MAX_FILESIZE"]
        mind = config["MINDAYS"]
        maxd = config["MAXDAYS"]
        
        # Get filesize
        stat = os.stat(os.path.join(config['UPLOAD_FOLDER'], filename))
        # Calculate how long the file should be alive for
        lifetime = mind + (-maxd + mind) * (stat.st_size / maxs - 1) ** 3
        # Add max age to current time to get the proper time it should delete
        deletiontime = datetime.now() + timedelta(days=lifetime)
        db.add_deletiontime(deletiontime, filename)
      else:
        deletiontime = time.time() + config["TIME"]
        deletiontime = datetime.fromtimestamp(deletiontime)
        db.add_deletiontime(deletiontime, filename)
        
      print_log('Main', 'New file processed "' + filename + '"')
      
      try:
        if request.form["source"] == "web":
          print_log('Web', 'Returned link page for "' + filename + '"', print_debug)
          return render_template('link.html', data=data, page=config["SITE_DATA"])
      except Exception:
        print_log('Web', 'No web reported in form, returned JSON', print_debug)
        return json.dumps(data)
    else:
      print_log('Notice', 'Forbidden file received')
      return error_page(error="This file isn't allowed, sorry!", code=403)

  # Return Web UI if we have a GET request
  elif request.method == 'GET':
    print_log('Web', 'Hit upload page', print_debug)
    return render_template('upload.html', page=config["SITE_DATA"])

# Def all the static pages
@app.route('/about')
def about():
  if config["METRICS"]:
    db.add_stat("aboutviews")
  return render_template('about.html', page=config["SITE_DATA"])
@app.route('/faq')
def faq():
  if config["METRICS"]:
    db.add_stat("faqviews")
  return render_template('faq.html', page=config["SITE_DATA"])
@app.route('/services')
def services():
  if config["METRICS"]:
    db.add_stat("servicesviews")
  return render_template('services.html', page=config["SITE_DATA"])
@app.route('/welcome')
def welcome():
  if config["METRICS"]:
    db.add_stat("welcomeviews")
  video = random.choice(os.listdir(config['THEME_FOLDER'] + "/static/welcome/"))
  return render_template('welcome.html', page=config["SITE_DATA"], video=video)
@app.route('/dmca')
def dmca():
  if config["METRICS"]:
    db.add_stat("dmcaviews")
  video = random.choice(os.listdir(config['THEME_FOLDER'] + "/static/dmca/"))
  return render_template('dmca.html', page=config["SITE_DATA"], video=video)
@app.route('/czb')
def czb():
  if config["METRICS"]:
    db.add_stat("czb")
  return render_template('czb.html', page=config["SITE_DATA"])
@app.route('/qr')
def qr():
  if config["METRICS"]:
    db.add_qrscan(request.environ.get('HTTP_X_FORWARDED_FOR', request.remote_addr))
  video = random.choice(os.listdir(config['THEME_FOLDER'] + "/static/welcome/"))
  return render_template('qr.html', page=config["SITE_DATA"], video=video)


# Static resources that browsers spam for
@app.route('/favicon.ico')
def favicon():
  return send_from_directory('static', 'favicon.ico')
@app.route('/apple-touch-icon.png')
def appleTouch():
  return send_from_directory('static', 'logo/152px.png')
@app.route('/robots.txt')
def robotsTxt():
  return send_from_directory('static', 'robots.txt')

# Custom 404
@app.errorhandler(404)
def page_not_found(e):
  db.add_stat("404")
  return error_page(error="We couldn't find that. Are you sure you know what you're looking for?", code=404), 404
@app.errorhandler(500)
def internal_error(e):
  db.add_stat("500")
  return error_page(error="Unknown error, try your upload again, or contact the admin.", code=500), 500
@app.errorhandler(403)
def no_permission(e):
  db.add_stat("403")
  return error_page(error="Permission denied, no snooping.", code=403), 403


@app.route('/<filename>', methods=['GET'])
def get_file(filename):
  print_log('Web', 'Hit "' + filename + '"')
  # try:
  db.update_file(filename)
  if config["METRICS"]:
    if '.' not in filename:
      db.add_fileview("None")
    else:
      db.add_fileview(filename.rsplit('.', 1)[1])
  # except Exception:
  #   print_log('Warning', 'Unable to update access time (and view) Is the file in the database?')
  if config["X-ACCEL-REDIRECT"]:
    r = make_response()
    r.headers['Content-Type'] = ''
    r.headers['X-Accel-Redirect'] = os.path.join("/" + config["UPLOAD_FOLDER"], filename)
    return r
  # return text/plain for files with no extensions
  if '.' not in filename:
    return send_from_directory(config['UPLOAD_FOLDER'], filename, mimetype="text/plain")
  # or if they're in the unsafe ext config.
  elif filename.endswith(config["UNSAFE_EXTENSIONS"]):
    return send_from_directory(config['UPLOAD_FOLDER'], filename, mimetype="text/plain")
  # Else, send and let flask guess the mimetype
  else:
    return send_from_directory(config['UPLOAD_FOLDER'], filename)

@app.route('/delete', methods=['POST'])
def delete_file_key_api():
  deletekey = (request.form['key'])
  if config["GEN_DELETEKEY"]:
    df = db.get_file_from_key(deletekey)
    if not df:
      print_log('Web', 'Someone used an invalid deletion key', print_debug)
      if config["METRICS"]:
        db.add_stat("invaliddeletionkey")
      return "Error: Invalid deletion key!\n"
    for file in df:
      print_log('Thread', 'Deleting ' + file["file"] + ' with deletion key')
      if config["METRICS"]:
        db.add_stat("urldeletions")
      try:
        os.remove(os.path.join(config["UPLOAD_FOLDER"], file["file"]))
        db.delete_entry(file["file"])
      except Exception:
        print_log('Warning', 'Failed to delete ' + file["file"])
        return "Error: Failed to delete file, not exactly sure why though\n"
  else:
    return 'Deletion keys are not enabled on this server.\n'
  return 'File deleted!\n'

@app.route('/delete/<deletekey>', methods=['GET'])
def delete_file_key(deletekey):
  if config["GEN_DELETEKEY"]:
    df = db.get_file_from_key(deletekey)
    if not df:
      print_log('Web', 'Someone used an invalid deletion key')
      if config["METRICS"]:
        db.add_stat("invaliddeletionkey")
      return error_page(error="Invalid deletion key, double check your key!", code=403), 403
    for file in df:
      print_log('Thread', 'Deleting ' + file["file"] + ' with deletion key')
      if config["METRICS"]:
        db.add_stat("urldeletions")
      try:
        os.remove(os.path.join(config["UPLOAD_FOLDER"], file["file"]))
        db.delete_entry(file["file"])
      except Exception:
        print_log('Warning', 'Failed to delete ' + file["file"] + ' : Invalid key')
        return error_page(error="Failed to delete file, just uhh.. go back to the main page", code=500), 500
  else:
    return error_page(error="Deletion keys are not enabled on this server.", code=403), 403
  return redirect('/')

# Configure nginx to use these urls as custom error pages
@app.route('/error/<int:error>')
def nginx_error(error):
  # out here doing 2iq plays because https://www.python.org/dev/peps/pep-0634/ isn't ready.
  if error == 413:
    return error_page(error="The file you uploaded was too large for the server to handle.", code=413), 413
  elif error == 403: # Block IPs with your web server and return /error/403 for this page
    return error_page(error="Permission Denied - No snooping allowed thanks :)", code=403), 403
  elif error == 404:
    return error_page(error="We couldn't find that. Are you sure you know what you're looking for?", code=404), 404
  elif error == 418:
    return error_page(error="ahaha teapot get it?", code=418), 418
  elif error == 500:
    return error_page(error="Internal server error, try again or contact the admin", code=500), 500
  else:
    return error_page(error="We literally have no idea what just happened, try again or contact the admin", code="Unknown")

# Metrics for Grafana
## This should be denied on the proxy, or disabled in config
## Try not to calculate anything on this, use the functions to update a table in
@app.route('/metrics/')
def metrics():
  if config["METRICS"]:
    response = make_response(generateMetrics(), 200)
    response.mimetype = "text/plain"
    return response
  return error_page(error="Metrics are disabled on this server :(", code=404), 404
if config["DELETE_FILES"]:
  cleaner_thread()

print_log('Main', 'Launching app', print_debug)
if __name__ == '__main__':
  app.run(
    port=config["PORT"],
    host=config["HOST"],
    debug=config["EXTENDED_DEBUG"]
  )