Initial move from Hyozan

Dominik V. Salonen 2015-12-23 14:11:38 +01:00
commit 7f8c57539c
13 changed files with 331 additions and 0 deletions

.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# JetBrains files
# Configuration
# Data

Hyozan/ Normal file
View File

Hyozan/ Normal file
View File

@ -0,0 +1,37 @@
import sqlite3
import time
def connect(target):
return sqlite3.connect(target)
def add_file(filename):
db = connect('files.db')
db.execute('INSERT INTO files (file, time, accessed) VALUES (?, ?, ?)',
[filename, time.time(), time.time()])
def update_file(filename):
db = connect('files.db')
db.execute('UPDATE files SET accessed = ? WHERE file = ?',
[time.time(), filename])
def add_b2(filename, file_id):
db = connect('files.db')
db.execute('UPDATE files SET b2 = ? WHERE file = ?',
[file_id, filename])
def check_value(column, value):
db = connect('files.db')
cur = db.execute('SELECT EXISTS(SELECT 1 FROM files WHERE ? = ?)', [column, value])
rv = cur.fetchone()
if rv:
return False
return True

Hyozan/ Normal file
View File

@ -0,0 +1,13 @@
from datetime import datetime
import time
def print_log(source, message):
if source == "Main":
print('\033[92m' + source + ': \033[0m' + message)
elif source == "Notice" or source == "Warning":
print('\033[93m' + source + ': \033[0m' + message)
print('\033[94m' + source + ': \033[0m' + message)
def time_to_string(unixtime):
return datetime.fromtimestamp(unixtime).strftime('%B %d, %Y (%H:%M - ' + time.tzname[time.daylight] + ')')

46 Normal file
View File

@ -0,0 +1,46 @@
# Hyozan
An "Object storage" like application which uses Backblaze B2 for archival. Originally intended for screenshot uploads via ShareX, but it will probably support other files as well.
Named after the Japanese word 氷山 (Hyōzan), meaning iceberg. Referencing how most of it is hidden underwater.
# The goal of this project
Bandwidth is overpriced. Really overpriced.
Don't get me wrong. B2's $0.05/GB is perfectly reasonable compared to all the others like S3 and Google Cloud. In fact, it's pretty good.
Problem is that they're **all** overpriced. Storage is cheap, but never use these services for bandwidth alone.
Why pay over $50 per TB of bandwidth when you can just install this on a VPS from a host like DigitalOcean that will give you the same for $5?
# What it does
When you upload a file to Hyozan, it forwards it to B2. When users then try to access it later, Hyozan will first check if it has a local copy. If it does not, it will fetch the file from B2 and keep it for a while.
This will decrease both the bandwidth and transaction (request) costs that come with object storage services.
## In technical terms
Hyozan is an Object storage oriented API-only (for now) reverse proxy, designed to cache, manage, adding and hopefully deleting static files from a 3rd party object storage service while lowering your total bandwidth consumption from these services, in this case B2. Due to the code structure, rewriting it for something like S3 should not be too hard.
# Why?
Instead of paying $50 for 1 TB of B2 bandwidth. Let us assume that you have a DO droplet running Hyozan. If it cached 90% of your traffic. **That means you only pay about $5 for B2 bandwidth and $5 for your droplet. That's a total of just $10. A measly fifth of the regular cost**
And this does not even consider transaction costs, which can be high if you serve a lot of smaller files.
# Requirements
* Python 3 (Python 2 might work, dunno, i don't test that, don't care either)
* Install flask, currently that should be the only requirement and hopefully forever (``pip install -r requirements.txt``)
# Using the thing
* Clone the repo somewhere
* Do ``cp``
* Edit ```` so that the information is correct
* If possible, make it listen on ```` and then use something like nginx as a reverse proxy. For security purposes
* ``chmod +x`` and then ``./``
* ???
* PROFIT (Hopefully)

0 Normal file
View File

25 Normal file
View File

@ -0,0 +1,25 @@
# Create an empty config dict
config = dict()
# Main server configuration
config["HOST"] = ""
# This string will be used in file URLs that are returned
config["DOMAIN"] = ""
config["PORT"] = 8282
config["DEBUG"] = True
# Extended debug will add extra debug output that's not normally provided by flask
config["EXTENDED_DEBUG"] = False
# Single user authentication, leave blank to disable authentication
config["KEY"] = ""
# File settings
config["UPLOAD_FOLDER"] = './data'
config["ALLOWED_EXTENSIONS"] = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
# Site info displayed to the user
config["SITE_DATA"] = {
"title": "QuadFile"

requirements.txt Normal file
View File

@ -0,0 +1,2 @@

90 Normal file
View File

@ -0,0 +1,90 @@
#!/usr/bin/env python3
from flask import Flask, Response, request, redirect, url_for, send_from_directory, abort, render_template
from werkzeug import secure_filename
from threading import Thread
import logging
import os
import json
import time
from random import randint
# Import our configuration
from conf import config
# Import Hyozan stuff
from Hyozan import db
from Hyozan.output import print_log, time_to_string
app = Flask(__name__)
# Pre-start functions
print_log('Main', 'Running authorization towards B2')
print_log('Main', 'Checking for data folder')
if not os.path.exists(config['UPLOAD_FOLDER']):
print_log('Main', 'Data folder not found, creating')
log = logging.getLogger('werkzeug')
def auth(key):
if config["KEY"] == "":
return True
elif config["KEY"] == key:
return True
return False
def allowed_file(filename):
return '.' in filename and filename.rsplit('.', 1)[1] in config["ALLOWED_EXTENSIONS"]
@app.route('/', methods=['GET', 'POST'])
def upload_file():
if request.method == 'POST':
if not auth(request.headers.get('X-Hyozan-Auth')):
data = dict()
file = request.files['file']
# Only continue if a file that's allowed gets submitted.
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
while os.path.exists(os.path.join(config["UPLOAD_FOLDER"], filename)):
filename = str(randint(1000,8999)) + '-' + secure_filename(filename)
thread1 = Thread(target = db.add_file, args = (filename,))
print_log('Thread', 'Adding to DB')['UPLOAD_FOLDER'], filename))
data["file"] = filename
data["url"] = config["DOMAIN"] + "/" + filename
print_log('Main', 'New file processed')
if request.form["source"] == "web":
return redirect(url_for('get_file', filename=filename), code=302)
return json.dumps(data)
# Return Web UI if we have a GET request
elif request.method == 'GET':
return render_template('upload.html', page=config["SITE_DATA"])
@app.route('/<filename>', methods=['GET'])
def get_file(filename):
print_log('Main', 'Hit "' + filename + '" - ' + time_to_string(time.time()))
return send_from_directory(config['UPLOAD_FOLDER'], filename)
if __name__ == '__main__':

schema.sql Normal file
View File

@ -0,0 +1,8 @@
-- noinspection SqlNoDataSourceInspectionForFile
drop table if exists files;
create table files (
file text primary key not null,
b2 text,
time int,
accessed int

static/style.css Normal file
View File

@ -0,0 +1,69 @@
@import url(,300,400,100);
/* Being lazy 101 */
* {
margin: 0;
padding: 0;
box-sizing: border-box;
body {
font-family: "Lato", sans-serif;
color: #ffffff;
background-color: #101010;
.header {
width: 100%;
padding: 10px;
.header h1 {
font-size: 36pt;
font-weight: 300;
text-align: center;
.header .inner {
width: 100%;
max-width: 920px;
margin-left: auto;
margin-right: auto;
.page {
width: 100%;
max-width: 920px;
padding: 10px;
margin-left: auto;
margin-right: auto;
.uploadForm {
width: 100%;
text-align: center;
.uploadForm input[type="file"] {
padding: 10px;
font-size: 12pt;
font-weight: 300;
border: none;
display: inline-block;
.uploadForm input[type="submit"] {
width: 100%;
padding: 20px;
font-size: 12pt;
font-weight: 300;
max-width: 320px;
background-color: #229922;
margin-top: 10px;
color: #FFFFFF;
border: none;
.uploadForm input[type="submit"]:hover {
background-color: #33BB33;

templates/layout.html Normal file
View File

@ -0,0 +1,21 @@
<!doctype html>
<title>{{ page.title }}</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
<link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
<meta name="viewport" content="width=device-width, initial-scale=1">
<div class="header">
<div class="inner">
{{ page.title }}
<div class="page">
{% block body %}{% endblock %}

templates/upload.html Normal file
View File

@ -0,0 +1,12 @@
{% extends "layout.html" %}
{% block body %}
<form action="" id="form" method="post" enctype="multipart/form-data" class="uploadForm">
<input id="file" type="file" name="file">
<input type="hidden" name="source" value="web">
document.getElementById("file").onchange = function() {
{% endblock %}