ghostifi-server-destroy-rebuild-now

server.py

import MySQLdb
import os
import sys
from sqlalchemy import create_engine, MetaData
from sqlalchemy.sql import text
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy import Column, String, Integer, Date, Table, ForeignKey, Float
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
from urllib import quote_plus as urlquote
from multiprocessing.dummy import Pool as ThreadPool 
import json
import urllib
import smtplib
from email.mime.text import MIMEText
import requests
import paramiko
import time

# Set PID file, this prevents the script from running if already running
pid = str(os.getpid())
pidfile = "/tmp/server-py.pid"
if os.path.isfile(pidfile):
    print "%s already exists, exiting" % pidfile
    sys.exit()
file(pidfile, 'w').write(pid)

try:
    # Global vars
    VULTR_API_KEY = "---"
    GHOSTIFI_SSH_KEY_ID = "---"
    RCHASE_SSH_KEY_ID = "---"
    SMTP_SSL_HOST = "---"
    SMTP_SSL_PORT = 465
    SMTP_USERNAME = "---"
    SMTP_PASSWORD = "---"
    SMTP_SENDER = "support@ghostifi.net"
    CF_ZONE = "---"
    CF_API_KEY = "---"
    CF_EMAIL ="---"
    CF_URL = 'https://api.cloudflare.com/client/v4/zones/' + CF_ZONE + '/dns_records'
    CF_HEADERS = {'Content-Type': 'application/json', 'X-Auth-Key': CF_API_KEY, 'X-Auth-Email': CF_EMAIL}
    SSH_PASSWORD = "---"
    DC_ID_DICT = {'New Jersey': '1', 'Chicago': '2', 'Dallas': '3', 'Seattle': '4', 'Los Angeles': '5', 'Atlanta': '6', 'Amsterdam': '7', 'London': '8', 'Frankfurt': '9', 'Silicon Valley': '12', 'Sydney': '19', 'Paris': '24', 'Tokyo': '25', 'Miami': '39', 'Singapore': '40'}
    SNAPSHOT_OS_ID = '164'
    DEFAULT_DC_ID = DC_ID_DICT["New Jersey"]
    DB_PASSWORD = '---'

    # Setup SQLAlchemy
    engine = create_engine('mysql://---:%s@localhost:3306/---' % urlquote(DB_PASSWORD), echo=False)
    metadata = MetaData(bind=engine)
    Session = scoped_session(sessionmaker(engine, autoflush=True))

    Base = declarative_base()

    # DB classes
    class Subscription(Base):
        __tablename__ = 'wp_edd_subscriptions'

        id = Column(Integer, primary_key=True)
        customer_id = Column(Integer)
        period = Column(String)
        initial_amount = Column(String)
        recurring_amount = Column(String)
        bill_times = Column(Integer)
        transaction_id = Column(String)
        parent_payment_id = Column(Integer)
        product_id = Column(Integer)
        created = Column(Date)
        expiration = Column(Date)
        trial_period = Column(String)
        status = Column(String)
        profile_id = Column(String)
        notes = Column(String)
        server = relationship("Server", uselist=False, backref="subscription")

    class Server(Base):
        __tablename__ = 'servers'

        id = Column(Integer, primary_key=True)
        customer_id = Column(Integer)
        product_id = Column(Integer)
        wp_edd_sub_id = Column(Integer, ForeignKey(Subscription.id))
        server_ip = Column(String)
        server_name = Column(String)
        email = Column(String)
        root_password = Column(String)
        bandwidth_this_month = Column(Float)
        bandwidth_limit_this_month = Column(Float)
        current_location = Column(String)
        rebuild_schedule = Column(String)
        rebuild_schedule_location = Column(String)
        rebuild_now_status = Column(Integer)
        rebuild_now_location = Column(String)
        ovpn_file = Column(String)
        status = Column(String)
        snapshot_id = Column(String)
        username = Column(String)
        vps_sub_id = Column(String)
        delete_request = Column(Integer)
        realtime_bandwidth_this_month = Column(Float)
        vps_plan_id = Column(String)

        def __init__(self, customer_id, product_id, wp_edd_sub_id, server_ip, server_name, email, root_password, ovpn_file, status, bandwidth_limit_this_month, vps_plan_id, username, vps_sub_id):
            self.vps_plan_id = vps_plan_id
            self.username = username
            self.customer_id = customer_id
            self.product_id = product_id
            self.wp_edd_sub_id = wp_edd_sub_id
            self.server_ip = server_ip
            self.server_name = "g0" + str(self.wp_edd_sub_id) + ".ghostifi.net"
            self.server_subdomain = "g0" + str(self.wp_edd_sub_id)
            self.email = email
            self.root_password = root_password
            self.bandwidth_this_month = 0
            self.bandwidth_limit_this_month = bandwidth_limit_this_month
            self.current_location = "New Jersey"
            self.rebuild_schedule = "None"
            self.rebuild_schedule_location = "New Jersey"
            self.rebuild_now_status = 0
            self.rebuild_now_location = "New Jersey"
            self.ovpn_file = 'GhostiFi_' + self.server_subdomain + '_UDP-993.ovpn' 
            self.status = status
            # Debian
            self.os_id = '244'
            self.vps_sub_id = ''
            self.snapshot_id = ''
            self.delete_request = 0
            self.realtime_bandwidth_this_month = 0.00

        def _create_vps(self):
            # Create new Vultr VPS: Debian 9, 1024MB, $5/mo, add rchase and ghostifi SSH keys
            print "Creating VPS..."
            url = 'https://api.vultr.com/v1/server/create'
            payload = {'hostname': self.server_name, 'label': self.server_name, 'DCID': DEFAULT_DC_ID,
                   'VPSPLANID': self.vps_plan_id, 'OSID': self.os_id,
                   'SSHKEYID': GHOSTIFI_SSH_KEY_ID + "," + RCHASE_SSH_KEY_ID}
            r = requests.post(url, data=payload, headers={"API-Key": VULTR_API_KEY})
            self.vps_sub_id = json.loads(r.text)["SUBID"]
            return self.vps_sub_id

        def _get_vps_status(self):
            print "Waiting for VPS to finish setup..."
            time_check = 0
            while time_check < 500:
                time_check += 1
                url = 'https://api.vultr.com/v1/server/list'
                r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
                server_state = json.loads(r.text)[self.vps_sub_id]["server_state"]
                if server_state != "none":
                    self.server_ip = json.loads(r.text)[self.vps_sub_id]["main_ip"]
                    break
                else:
                    time.sleep(1)
            return server_state

        def _get_root_password(self):
            print "Getting root password..."
            url = 'https://api.vultr.com/v1/server/list'
            r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
            self.root_password = json.loads(r.text)[self.vps_sub_id]["default_password"]
            if self.root_password != None:
                print "Root password saved!"
                result = 1
            else:
                print "Error retrieving root password"
                result = 0
                time.sleep(1)
            return result

        def _create_cf_dns(self):
            global CF_URL
            print "Creating Cloudflare DNS record..."
            cf_payload = {'type': 'A', 'name': self.server_name, 'content': self.server_ip, 'ttl': 120}
            r = requests.post(CF_URL, data=json.dumps(cf_payload), headers=CF_HEADERS)
            json_data = json.loads(r.text)
            print r.text
            try:
              if json_data["result"]["id"]:
                return json_data["result"]["id"]
            except:
              return 0

        def _install_openvpn(self):
            print "Installing OpenVPN..."
            k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
            c = paramiko.SSHClient()
            c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            times_tried = 0
            while times_tried < 30:
                try:
                    c.connect(hostname=self.server_ip, username="root", pkey=k)
                    ssh_connected = 1
                    break
                except Exception as e:
                    print e
                    time.sleep(3)
                    times_tried += 1
                    print "Trying to connect to SSH again..."
                    ssh_connected = 0
            if ssh_connected == 0:
                return 0
            commands = ['wget https://raw.githubusercontent.com/GhostiFi/openvpn-install.sh/master/openvpn-install.sh', 'chmod +x /root/openvpn-install.sh', '/bin/bash /root/openvpn-install.sh']
            for command in commands:
                time.sleep(1)
                # print "-----"
                print "Executing {}".format(command)
                stdin, stdout, stderr = c.exec_command(command)
                output = stdout.read()
                # print output
                # print "Errors:"
                errors = stderr.read()
                # print errors
                # print "-----"
            c.close()

        def _get_ovpn_file(self):
            print "Moving OVPN file to webserver..."
            k = paramiko.RSAKey.from_private_key_file("/home/ghostifi/ghostifi.pem", password=SSH_PASSWORD)
            c = paramiko.SSHClient()
            c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            c.connect(hostname=self.server_ip, username="root", pkey=k)
            sftp_client = c.open_sftp()
            sftp_client.get('/root/GhostiFi.ovpn','/home/ghostifi/ovpn/' + self.ovpn_file)
            sftp_client.close()

        def _create_vps_snapshot(self):
            print "Creating VPS snapshot..."
            vps_plan_id =  self.vps_plan_id
            url = 'https://api.vultr.com/v1/snapshot/create'
            payload = {'SUBID': self.vps_sub_id}
            r = requests.post(url, data=payload, headers={"API-Key": VULTR_API_KEY})
            self.snapshot_id = json.loads(r.text)["SNAPSHOTID"]
            return self.vps_sub_id

        def _send_email(self, targets, msg_txt, subject):
            print "Sending email..."
            msg = MIMEText(msg_txt)
            msg['Subject'] = subject
            msg['From'] = SMTP_SENDER
            msg['To'] = ', '.join(targets)
            server = smtplib.SMTP_SSL(SMTP_SSL_HOST, SMTP_SSL_PORT)
            server.login(SMTP_USERNAME, SMTP_PASSWORD)
            server.sendmail(SMTP_SENDER, targets, msg.as_string())
            server.quit()
            print "Email sent!"

        def _update_bandwidth_this_month(self, vps_sub_id):
            print "Updating bandwidth this month..."
            url = 'https://api.vultr.com/v1/server/list'
            r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
            current_bandwidth = json.loads(r.text)[vps_sub_id]["current_bandwidth_gb"]
            self.bandwidth_this_month += current_bandwidth
            print "Bandwidth this month updated!"

        def _update_realtime_bandwidth_this_month(self):
            print "Updating realtime bandwidth this month..."
            url = 'https://api.vultr.com/v1/server/list'
            r = requests.get(url, headers={"API-Key": VULTR_API_KEY})
            current_bandwidth = json.loads(r.text)[self.vps_sub_id]["current_bandwidth_gb"]
            self.realtime_bandwidth_this_month = self.bandwidth_this_month + current_bandwidth
            print "Realtime bandwidth this month updated!"

        def _create_vps_from_snapshot(self):
            # Create new Vultr VPS from this user's original snapshot
            print "Creating VPS from snapshot..."
            vps_plan_id =  self.vps_plan_id
            url = 'https://api.vultr.com/v1/server/create'
            payload = {'hostname': self.server_name, 'label': self.server_name, 'DCID': DC_ID_DICT[self.rebuild_now_location],
                   'VPSPLANID': self.vps_plan_id, 'OSID': SNAPSHOT_OS_ID,
                   'SSHKEYID': GHOSTIFI_SSH_KEY_ID + "," + RCHASE_SSH_KEY_ID, 'SNAPSHOTID': self.snapshot_id}
            r = requests.post(url, data=payload, headers={"API-Key": VULTR_API_KEY})
            print r.text
            if "Snapshot cannot be used at this time" in r.text:
                # Set rebuild_now_status back to 1 so it will try again later
                self.rebuild_now_status = 1
                return 0
            else:
                self.vps_sub_id = json.loads(r.text)["SUBID"]
                return self.vps_sub_id

        def _destroy_vps(self, vps_sub_id):
            print "Destroying VPS..."
            url = 'https://api.vultr.com/v1/server/destroy'
            payload = {'SUBID': vps_sub_id}
            r = requests.post(url, data=payload, headers={"API-Key": VULTR_API_KEY})
            print r

        def _delete_ovpn_file(self):
            if os.path.exists("/home/ghostifi/ovpn/" + self.ovpn_file):
              os.remove("/home/ghostifi/ovpn/" + self.ovpn_file)
            else:
              print("The OVPN file does not exist!")

        def _get_cf_record_id(self):
            global CF_URL
            r = requests.get(CF_URL, headers=CF_HEADERS, params={'per_page':1000})
            json_data = json.loads(r.text)
            for row in json_data["result"]:
                if row["name"] == self.server_name:
                    return row["id"]
            return 0

        def _update_cf_dns(self):
            print "Updating Cloudflare DNS..."
            global CF_URL
            global CF_HEADERS
            cf_record_id = self._get_cf_record_id()
            CF_URL += "/" + cf_record_id
            cf_payload = {'type': 'A', 'name': self.server_name, 'content': self.server_ip, 'ttl': 120}
            r = requests.put(CF_URL, data=json.dumps(cf_payload), headers=CF_HEADERS)
            json_data = json.loads(r.text)
            print r.text
            return 1

        def _destroy_snapshot(self, snapshot_id):
            print "Destroying snapshot..."
            url = 'https://api.vultr.com/v1/snapshot/destroy'
            payload = {'SNAPSHOTID': snapshot_id}
            r = requests.post(url, data=payload, headers={"API-Key": VULTR_API_KEY})
            print r

        def _delete_cf_dns(self):
            global CF_URL
            global CF_HEADERS
            cf_record_id = self._get_cf_record_id()
            CF_URL += "/" + cf_record_id
            r = requests.delete(CF_URL, headers=CF_HEADERS)
            json_data = json.loads(r.text)
            print json_data

        def create(self):
            self._create_vps()
            vps_status = self._get_vps_status()
            if vps_status != 'none':
                print "VPS setup complete!"
                cf_create_status = self._create_cf_dns()
                if cf_create_status != 0:
                    print "CF DNS record created!"
                    self._install_openvpn()
                    self._get_ovpn_file()
                    self._create_vps_snapshot()
                    self._get_root_password()
                    self.status = 'Running'
                    # Send setup complete notification email
                    targets = [SMTP_SENDER, self.email]
                    msg_txt = 'Thanks for using GhostiFi!\nYour VPS VPN has finished installing. Login at https://ghostifi.net/user to find your OVPN file, as well as instructions on how to get started connecting devices.\n\nUsername: ' + self.username + '\n' + 'Server: ' + self.server_name + '\n\nIf you need any help just reply to this email, and we will get back to you shortly\n\n'
                    subject = 'Your VPS VPN is ready'
                    self._send_email(targets, msg_txt, subject)
                    print "Your VPS VPN is ready!"
                else:
                    print "CF DNS record create failed."

            else:
                print "VPS setup failed."

        def destroy(self):
            print "Destroying VPS VPN..."
            self._destroy_vps(self.vps_sub_id)
            self._destroy_snapshot(self.snapshot_id)
            self._delete_cf_dns()
            self._delete_ovpn_file()
            print "VPS VPN destroyed!"

        def rebuild_now(self):
            old_vps_sub_id = self.vps_sub_id
            vps_status = self._create_vps_from_snapshot()
            if vps_status == 0:
                print "VPS setup failed, try again later."
                self.rebuild_now_status = 1
                self.status = 'Running'

            else:
                time.sleep(15)
                while True:
                    vps_status = self._get_vps_status()
                    print vps_status
                    if vps_status != 'ok':
                        time.sleep(3)
                        continue
                    else:
                        break
        
                if vps_status != 'none':
                    print "VPS rebuild complete!"
                    cf_update_status = self._update_cf_dns()
                    if cf_update_status != 0:
                        print "CF DNS record updated!"
                        self._update_bandwidth_this_month(old_vps_sub_id)
                        self._destroy_vps(old_vps_sub_id)
                        self.rebuild_now_status = 0
                        self.status = 'Running'

                    else:
                        print "CF DNS record update failed."
                else:
                    print "VPS rebuild failed."

    # Setup SQLAlchemy stuff
    Base.metadata.create_all(engine)
    session = Session()

    # Create lists which will be populated by SQL queries
    servers_to_create = []
    servers_to_destroy = []

    # Get all active subscriptions, joined to servers
    active_subscriptions = session.query(Subscription, Server).outerjoin(Server, Subscription.id == Server.wp_edd_sub_id).filter(Subscription.status=="Active").all()
    
    # Find active subscriptions which do not have existing servers (create these)
    for subscription in active_subscriptions:
        # If subscription exists for this server, skip, else append to the server create list
        if subscription[1]:
            pass
        else:
            servers_to_create.append(subscription[0])

    # Get all existing servers, joined to subscriptions
    active_servers = session.query(Server, Subscription).outerjoin(Subscription, Subscription.id == Server.wp_edd_sub_id).all()

    # Find existing servers which do not have active subscriptions (destroy these)
    for server in active_servers:
        # If subscription exists for this server, skip, else append to the server destroy list
        if server[1]:
            pass
        else:
            servers_to_destroy.append(server[0])

    # Get servers marked for delete
    delete_request_servers = session.query(Server, Subscription).outerjoin(Subscription, Subscription.id == Server.wp_edd_sub_id).filter(Server.delete_request == 1).all()

    for server in delete_request_servers:
        servers_to_destroy.append(server[0])

    # Get all servers which need to be rebuilt now (rebuild these)
    servers_to_rebuild_now = session.query(Server).filter(Server.rebuild_now_status==1).all()

    # Make the Pool of workers
    pool = ThreadPool(4)

    def thread_servers_to_create(self):
        # Setup SQLAlchemy
        session = Session()
        # If product is The VPS VPN Monthly or Yearly, set the VPS plan_id to 201 (25GB SSD/1GB RAM)
        if self.product_id == 5747 or self.product_id == 5745:
            vps_plan_id = '201'
            bandwidth_limit_this_month = 1000.00

        # Get username, email out of wp_users table. Using a raw query because this is a one-off
        sql = text('Select wp_users.user_login, wp_users.user_email from wp_edd_subscriptions left join wp_edd_customers on wp_edd_subscriptions.customer_id = wp_edd_customers.id left join wp_users on wp_edd_customers.user_id = wp_users.id where wp_edd_subscriptions.id = :id')
        query = session.execute(sql, {'id': self.id}).fetchone()
        username = query[0]
        email = query[1]
        # Create a new server object
        new_server = Server(self.customer_id, self.product_id, self.id, '0.0.0.0', '', email, '', '', 'Installing', bandwidth_limit_this_month, vps_plan_id, username, '')
        # Save to database
        session.add(new_server)
        session.commit()
        new_server.create()
        session.commit()
        # Close session
        session.close()

    def thread_servers_to_destroy(self):
        # Change status to Rebuilding to display this in the user dashboard
        self.status = 'Destroying'
        session.add(self)
        # Update database set status from Running to Rebuilding
        session.commit()
        # Run rebuild function
        self.destroy()
        # Commit changes - update status, rebuild_location, bandwidth_this_month etc
        session.delete(self)
        session.commit()

    def thread_servers_to_rebuild_now(self):
        # Change status to Rebuilding to display this in the user dashboard
        self.status = 'Rebuilding'
        self.rebuild_now_status = 0
        session.add(self)
        # Update database set status from Running to Rebuilding
        session.commit()
        # Run rebuild function
        self.rebuild_now()
        # Commit changes - update status, rebuild_location, bandwidth_this_month etc
        session.add(self)
        session.commit()

    # Start a thread for each servers_to_create
    results = pool.map(thread_servers_to_create, servers_to_create)

    # Start a thread for each servers_to_destroy
    results = pool.map(thread_servers_to_destroy, servers_to_destroy)

    # Start a thread for each servers_to_rebuild_now
    results = pool.map(thread_servers_to_rebuild_now, servers_to_rebuild_now)

    # close the pool and wait for the work to finish
    pool.close()
    pool.join()
    session.close()

finally:
    os.unlink(pidfile)