Skip to content

Instantly share code, notes, and snippets.

@reillychase
Created November 29, 2018 18:50
Embed
What would you like to do?
ghostifi-digitalocean-spaces-integration
import MySQLdb
import os
import sys
from sqlalchemy import create_engine, MetaData
from sqlalchemy.sql import text
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy import Column, String, Integer, Date, Table, ForeignKey, Float
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
from urllib import quote_plus as urlquote
from multiprocessing.dummy import Pool as ThreadPool
import json
import urllib
import smtplib
from email.mime.text import MIMEText
import requests
import paramiko
import time
from config import *
import boto3
# Set PID file, this prevents the script from running if already running
pid = str(os.getpid())
pidfile = "/tmp/server-py.pid"
if os.path.isfile(pidfile):
print "%s already exists, exiting" % pidfile
sys.exit()
file(pidfile, 'w').write(pid)
try:
# Setup SQLAlchemy
engine = create_engine('mysql://ghostifidbuser:%s@localhost:3306/ghostifi' % urlquote(DB_PASSWORD), echo=False)
metadata = MetaData(bind=engine)
Session = scoped_session(sessionmaker(engine, autoflush=True))
Base = declarative_base()
# DB classes
class Subscription(Base):
__tablename__ = 'wp_edd_subscriptions'
id = Column(Integer, primary_key=True)
customer_id = Column(Integer)
period = Column(String)
initial_amount = Column(String)
recurring_amount = Column(String)
bill_times = Column(Integer)
transaction_id = Column(String)
parent_payment_id = Column(Integer)
product_id = Column(Integer)
created = Column(Date)
expiration = Column(Date)
trial_period = Column(String)
status = Column(String)
profile_id = Column(String)
notes = Column(String)
server = relationship("Server", uselist=False, backref="subscription")
class Server(Base):
__tablename__ = 'servers'
id = Column(Integer, primary_key=True)
customer_id = Column(Integer)
product_id = Column(Integer)
wp_edd_sub_id = Column(Integer, ForeignKey(Subscription.id))
server_ip = Column(String)
server_name = Column(String)
email = Column(String)
root_password = Column(String)
bandwidth_this_month = Column(Float)
bandwidth_limit_this_month = Column(Float)
current_location = Column(String)
rebuild_schedule = Column(String)
rebuild_schedule_location = Column(String)
rebuild_now_status = Column(Integer)
rebuild_now_location = Column(String)
ovpn_file = Column(String)
status = Column(String)
username = Column(String)
vps_sub_id = Column(String)
delete_request = Column(Integer)
realtime_bandwidth_this_month = Column(Float)
vps_plan_id = Column(String)
do_file = Column(String)
def __init__(self, customer_id, product_id, wp_edd_sub_id, server_ip, server_name, email, root_password, ovpn_file, status, bandwidth_limit_this_month, vps_plan_id, username, vps_sub_id):
self.vps_plan_id = vps_plan_id
self.username = username
self.customer_id = customer_id
self.product_id = product_id
self.wp_edd_sub_id = wp_edd_sub_id
self.server_ip = server_ip
self.server_name = "g0" + str(self.wp_edd_sub_id) + ".ghostifi.net"
self.server_subdomain = "g0" + str(self.wp_edd_sub_id)
self.email = email
self.root_password = root_password
self.bandwidth_this_month = 0
self.bandwidth_limit_this_month = bandwidth_limit_this_month
self.current_location = "New Jersey"
self.rebuild_schedule = "None"
self.rebuild_schedule_location = "New Jersey"
self.rebuild_now_status = 0
self.rebuild_now_location = "New Jersey"
self.ovpn_file = 'GhostiFi_' + self.server_subdomain + '_UDP-993.ovpn'
self.status = status
self.vps_sub_id = ''
self.delete_request = 0
self.realtime_bandwidth_this_month = 0.00
self.do_file = self.server_subdomain + '.tar.gz'
def _create_vps(self, type):
if type == 'new':
location = DEFAULT_DC_ID
elif type == 'rebuild':
location = DC_ID_DICT[self.rebuild_now_location]
# Create new Vultr VPS: Debian 9, 1024MB, $5/mo, add rchase and ghostifi SSH keys
print "Creating VPS..."
url = 'https://api.vultr.com/v1/server/create'
payload = {'hostname': self.server_name, 'label': self.server_name, 'DCID': location,
'VPSPLANID': self.vps_plan_id, 'OSID': OS_ID,
'SSHKEYID': GHOSTIFI_SSH_KEY_ID + "," + RCHASE_SSH_KEY_ID}
r = requests.post(url, data=payload, headers={"API-Key": VULTR_API_KEY})
self.vps_sub_id = json.loads(r.text)["SUBID"]
return self.vps_sub_id
def _get_vps_status(self):
print "Waiting for VPS to finish setup..."
time_check = 0
while time_check < 500:
time_check += 1
url = 'https://api.vultr.com/v1/server/list'
r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
server_state = json.loads(r.text)[self.vps_sub_id]["server_state"]
if server_state != "none":
self.server_ip = json.loads(r.text)[self.vps_sub_id]["main_ip"]
break
else:
time.sleep(1)
return server_state
def _get_root_password(self):
print "Getting root password..."
url = 'https://api.vultr.com/v1/server/list'
r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
self.root_password = json.loads(r.text)[self.vps_sub_id]["default_password"]
if self.root_password != None:
print "Root password saved!"
result = 1
else:
print "Error retrieving root password"
result = 0
time.sleep(1)
return result
def _create_cf_dns(self):
global CF_URL
print "Creating Cloudflare DNS record..."
cf_payload = {'type': 'A', 'name': self.server_name, 'content': self.server_ip, 'ttl': 120}
r = requests.post(CF_URL, data=json.dumps(cf_payload), headers=CF_HEADERS)
json_data = json.loads(r.text)
print r.text
try:
if json_data["result"]["id"]:
return json_data["result"]["id"]
except:
return 0
def _install_openvpn(self):
print "Installing OpenVPN..."
k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
times_tried = 0
while times_tried < 30:
try:
c.connect(hostname=self.server_ip, username="root", pkey=k)
ssh_connected = 1
break
except Exception as e:
print e
time.sleep(3)
times_tried += 1
print "Trying to connect to SSH again..."
ssh_connected = 0
if ssh_connected == 0:
return 0
commands = ['wget https://raw.githubusercontent.com/GhostiFi/openvpn-install.sh/master/openvpn-install.sh', 'chmod +x /root/openvpn-install.sh', '/bin/bash /root/openvpn-install.sh']
for command in commands:
time.sleep(1)
# print "-----"
print "Executing {}".format(command)
stdin, stdout, stderr = c.exec_command(command)
output = stdout.read()
# print output
# print "Errors:"
errors = stderr.read()
# print errors
# print "-----"
c.close()
def _get_ovpn_file(self):
print "Moving OVPN file to webserver..."
k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(hostname=self.server_ip, username="root", pkey=k)
sftp_client = c.open_sftp()
sftp_client.get('/root/GhostiFi.ovpn','/home/ghostifi/ovpn/' + self.ovpn_file)
sftp_client.close()
def _send_email(self, targets, msg_txt, subject):
print "Sending email..."
msg = MIMEText(msg_txt)
msg['Subject'] = subject
msg['From'] = SMTP_SENDER
msg['To'] = ', '.join(targets)
server = smtplib.SMTP_SSL(SMTP_SSL_HOST, SMTP_SSL_PORT)
server.login(SMTP_USERNAME, SMTP_PASSWORD)
server.sendmail(SMTP_SENDER, targets, msg.as_string())
server.quit()
print "Email sent!"
def _update_bandwidth_this_month(self, vps_sub_id):
print "Updating bandwidth this month..."
url = 'https://api.vultr.com/v1/server/list'
r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
current_bandwidth = json.loads(r.text)[vps_sub_id]["current_bandwidth_gb"]
self.bandwidth_this_month += current_bandwidth
print "Bandwidth this month updated!"
def _update_realtime_bandwidth_this_month(self):
print "Updating realtime bandwidth this month..."
url = 'https://api.vultr.com/v1/server/list'
r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
current_bandwidth = json.loads(r.text)[self.vps_sub_id]["current_bandwidth_gb"]
self.realtime_bandwidth_this_month = self.bandwidth_this_month + current_bandwidth
print "Realtime bandwidth this month updated!"
def _destroy_vps(self, vps_sub_id):
print "Destroying VPS..."
url = 'https://api.vultr.com/v1/server/destroy'
payload = {'SUBID': vps_sub_id}
r = requests.post(url, data=payload, headers={"API-Key": VULTR_API_KEY})
print r
def _delete_ovpn_file(self):
if os.path.exists("/home/ghostifi/ovpn/" + self.ovpn_file):
os.remove("/home/ghostifi/ovpn/" + self.ovpn_file)
def _get_cf_record_id(self):
global CF_URL
r = requests.get(CF_URL, headers=CF_HEADERS, params={'per_page':1000})
json_data = json.loads(r.text)
for row in json_data["result"]:
if row["name"] == self.server_name:
return row["id"]
return 0
def _update_cf_dns(self):
print "Updating Cloudflare DNS..."
global CF_URL
global CF_HEADERS
cf_record_id = self._get_cf_record_id()
CF_URL += "/" + str(cf_record_id)
cf_payload = {'type': 'A', 'name': self.server_name, 'content': self.server_ip, 'ttl': 120}
r = requests.put(CF_URL, data=json.dumps(cf_payload), headers=CF_HEADERS)
json_data = json.loads(r.text)
print r.text
return 1
def _tar_dir(self):
print "Tarring..."
k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
times_tried = 0
while times_tried < 30:
try:
c.connect(hostname=self.server_ip, username="root", pkey=k)
ssh_connected = 1
break
except Exception as e:
print e
time.sleep(1)
times_tried += 1
print "Trying to connect to SSH again..."
ssh_connected = 0
if ssh_connected == 0:
return 0
commands = ['tar cvzf ' + self.server_subdomain + '.tar.gz ' + OPENVPN_DIR]
for command in commands:
time.sleep(1)
# print "-----"
print "Executing {}".format(command)
stdin, stdout, stderr = c.exec_command(command)
output = stdout.read()
# print output
# print "Errors:"
errors = stderr.read()
# print errors
# print "-----"
c.close()
return self.do_file
def _save_to_do_spaces(self):
print "Moving tar file to webserver..."
k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(hostname=self.server_ip, username="root", pkey=k)
sftp_client = c.open_sftp()
sftp_client.get('/root/' + self.do_file,'/home/ghostifi/' + self.do_file)
sftp_client.close()
print "Moving tar to DO Space..."
boto3_session = boto3.session.Session()
do_client = boto3_session.client('s3',
region_name='nyc3',
endpoint_url='https://nyc3.digitaloceanspaces.com',
aws_access_key_id=DO_API_PUBLIC,
aws_secret_access_key=DO_API_SECRET)
try:
response = do_client.put_object(ACL='private', Bucket='ghostifi', Key=self.do_file, Body=open("/home/ghostifi/" + self.do_file,"rb"), Metadata={'server': self.server_name})
if os.path.exists("/home/ghostifi/" + self.do_file):
print "Removing tar from webserver..."
os.remove("/home/ghostifi/" + self.do_file)
return 1
except Exception as e:
print e
if os.path.exists("/home/ghostifi/" + self.do_file):
print "Removing tar from webserver..."
os.remove("/home/ghostifi/" + self.do_file)
return 0
def _download_from_do_spaces(self):
print "Downloading x.tar.gz to webserver..."
boto3_session = boto3.session.Session()
do_client = boto3_session.client('s3',
region_name='nyc3',
endpoint_url='https://nyc3.digitaloceanspaces.com',
aws_access_key_id=DO_API_PUBLIC,
aws_secret_access_key=DO_API_SECRET)
try:
# The two arguments are Key, local filename
do_client.download_file(Bucket="ghostifi", Key=self.do_file, Filename='/home/ghostifi/' + self.do_file)
except Exception as e:
print e
print "Moving tar file to VPN server..."
k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(hostname=self.server_ip, username="root", pkey=k)
sftp_client = c.open_sftp()
source = '/home/ghostifi/' + self.do_file
destination = '/root/' + self.do_file
sftp_client.put(source, destination)
sftp_client.close()
if os.path.exists("/home/ghostifi/" + self.do_file):
os.remove("/home/ghostifi/" + self.do_file)
def _untar_dir(self):
print "Untarring..."
k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
times_tried = 0
while times_tried < 30:
try:
c.connect(hostname=self.server_ip, username="root", pkey=k)
ssh_connected = 1
break
except Exception as e:
print e
time.sleep(1)
times_tried += 1
print "Trying to connect to SSH again..."
ssh_connected = 0
if ssh_connected == 0:
return 0
commands = ['rm -rf /etc/openvpn', 'mkdir /etc/openvpn', 'tar xzf ' + self.do_file + ' --directory /', 'rm /root/' + self.do_file, 'service openvpn restart', 'update-rc.d openvpn defaults']
for command in commands:
time.sleep(1)
# print "-----"
print "Executing {}".format(command)
stdin, stdout, stderr = c.exec_command(command)
output = stdout.read()
print output
print "Errors:"
errors = stderr.read()
print errors
print "-----"
c.close()
return 1
def _delete_cf_dns(self):
global CF_URL
global CF_HEADERS
cf_record_id = self._get_cf_record_id()
CF_URL += "/" + cf_record_id
r = requests.delete(CF_URL, headers=CF_HEADERS)
json_data = json.loads(r.text)
print json_data
def _delete_from_do_spaces(self):
print "Deleting tar.gz from DO Spaces..."
boto3_session = boto3.session.Session()
do_client = boto3_session.client('s3',
region_name='nyc3',
endpoint_url='https://nyc3.digitaloceanspaces.com',
aws_access_key_id=DO_API_PUBLIC,
aws_secret_access_key=DO_API_SECRET)
try:
# The two arguments are Key, local filename
do_client.delete_object(Bucket="ghostifi", Key=self.do_file)
except Exception as e:
print e
def create(self):
self._create_vps('new')
vps_status = self._get_vps_status()
if vps_status != 'none':
print "VPS setup complete!"
cf_create_status = self._create_cf_dns()
if cf_create_status != 0:
print "CF DNS record created!"
self._install_openvpn()
self._get_ovpn_file()
self._get_root_password()
self._tar_dir()
self._save_to_do_spaces()
self.status = 'Running'
# Send setup complete notification email
targets = [SMTP_SENDER, self.email]
msg_txt = 'Thanks for using GhostiFi!\nYour VPS VPN has finished installing. Login at https://ghostifi.net/user to find your OVPN file, as well as instructions on how to get started connecting devices.\n\nUsername: ' + self.username + '\n' + 'Server: ' + self.server_name + '\n\nIf you need any help just reply to this email, and we will get back to you shortly\n\n'
subject = 'Your VPS VPN is ready'
self._send_email(targets, msg_txt, subject)
print "Your VPS VPN is ready!"
else:
print "CF DNS record create failed."
else:
print "VPS setup failed."
def destroy(self):
print "Destroying VPS VPN..."
self._destroy_vps(self.vps_sub_id)
self._delete_cf_dns()
self._delete_ovpn_file()
self._delete_from_do_spaces()
print "VPS VPN destroyed!"
def rebuild_now(self):
old_vps_sub_id = self.vps_sub_id
# Create new VPS, install OpenVPN
self._create_vps('rebuild')
vps_status = self._get_vps_status()
if vps_status != 'none':
print "VPS setup complete!"
self._install_openvpn()
self._download_from_do_spaces()
self._untar_dir()
self._get_root_password()
cf_update_status = self._update_cf_dns()
if cf_update_status != 0:
print "CF DNS record updated!"
self.rebuild_now_status = 0
self.status = 'Running'
# Send rebuild complete notification email
targets = [SMTP_SENDER, self.email]
msg_txt = 'Thanks for using GhostiFi!\nYour VPS VPN has finished rebuilding. Your devices should reconnect automatically in the next few minutes.\n\nIf you need any help just reply to this email, and we will get back to you shortly\n\n'
subject = 'Your VPS VPN finished rebuilding'
self._send_email(targets, msg_txt, subject)
print "Your VPS VPN has finished rebuilding!"
# Wait .5 minutes for DNS to propagate before destroying old VPS
time.sleep(30)
self._update_bandwidth_this_month(old_vps_sub_id)
self._destroy_vps(old_vps_sub_id)
else:
print "CF DNS record update failed."
else:
print "VPS setup failed, try again later."
self.rebuild_now_status = 1
self.status = 'Running'
# Setup SQLAlchemy stuff
Base.metadata.create_all(engine)
session = Session()
# Create lists which will be populated by SQL queries
servers_to_create = []
servers_to_destroy = []
# Get all active subscriptions, joined to servers
active_subscriptions = session.query(Subscription, Server).outerjoin(Server, Subscription.id == Server.wp_edd_sub_id).filter(Subscription.status=="Active").all()
# Find active subscriptions which do not have existing servers (create these)
for subscription in active_subscriptions:
# If subscription exists for this server, skip, else append to the server create list
if subscription[1]:
pass
else:
servers_to_create.append(subscription[0])
# Get all existing servers, joined to subscriptions
active_servers = session.query(Server, Subscription).outerjoin(Subscription, Subscription.id == Server.wp_edd_sub_id).all()
# Find existing servers which do not have active subscriptions (destroy these)
for server in active_servers:
# If subscription exists for this server, skip, else append to the server destroy list
if server[1]:
pass
else:
servers_to_destroy.append(server[0])
# Get servers marked for delete
delete_request_servers = session.query(Server, Subscription).outerjoin(Subscription, Subscription.id == Server.wp_edd_sub_id).filter(Server.delete_request == 1).all()
for server in delete_request_servers:
servers_to_destroy.append(server[0])
# Get all servers which need to be rebuilt now (rebuild these)
servers_to_rebuild_now = session.query(Server).filter(Server.rebuild_now_status==1).all()
# Make the Pool of workers
pool = ThreadPool(4)
def thread_servers_to_create(self):
# Setup SQLAlchemy
session = Session()
# If product is The VPS VPN Monthly or Yearly, set the VPS plan_id to 201 (25GB SSD/1GB RAM)
if self.product_id == 5747 or self.product_id == 5745:
vps_plan_id = '201'
bandwidth_limit_this_month = 1000.00
# Get username, email out of wp_users table. Using a raw query because this is a one-off
sql = text('Select wp_users.user_login, wp_users.user_email from wp_edd_subscriptions left join wp_edd_customers on wp_edd_subscriptions.customer_id = wp_edd_customers.id left join wp_users on wp_edd_customers.user_id = wp_users.id where wp_edd_subscriptions.id = :id')
query = session.execute(sql, {'id': self.id}).fetchone()
username = query[0]
email = query[1]
# Create a new server object
new_server = Server(self.customer_id, self.product_id, self.id, '0.0.0.0', '', email, '', '', 'Installing', bandwidth_limit_this_month, vps_plan_id, username, '')
# Save to database
session.add(new_server)
session.commit()
new_server.create()
session.commit()
# Close session
session.close()
def thread_servers_to_destroy(self):
# Change status to Rebuilding to display this in the user dashboard
self.status = 'Destroying'
session.add(self)
# Update database set status from Running to Rebuilding
session.commit()
# Run rebuild function
self.destroy()
# Commit changes - update status, rebuild_location, bandwidth_this_month etc
session.delete(self)
session.commit()
def thread_servers_to_rebuild_now(self):
# Change status to Rebuilding to display this in the user dashboard
self.status = 'Rebuilding'
self.rebuild_now_status = 0
session.add(self)
# Update database set status from Running to Rebuilding
session.commit()
# Run rebuild function
self.rebuild_now()
# Commit changes - update status, rebuild_location, bandwidth_this_month etc
session.add(self)
session.commit()
# Start a thread for each servers_to_create
results = pool.map(thread_servers_to_create, servers_to_create)
# Start a thread for each servers_to_destroy
results = pool.map(thread_servers_to_destroy, servers_to_destroy)
# Start a thread for each servers_to_rebuild_now
results = pool.map(thread_servers_to_rebuild_now, servers_to_rebuild_now)
# close the pool and wait for the work to finish
pool.close()
pool.join()
session.close()
finally:
os.unlink(pidfile)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment