nginx_conf_auto_deploy.py

#!/bin/env python
#-*- coding: utf-8 -*-

""" 发布特定分支的 Nginx conf, 这里发布以 autodeploy 开头的分支.

此脚本跑在 Jenkins 任务下, 此任务通过 Jenkins 的 Gerrit Trigger
插件监控 Nginx conf (Gerrit 项目) 的 Ref Updated 和 Change Merged 事件,
而且分支要匹配 **/autodeploy*, 满足条件触发此任务.

1. 先计算出最近一次修改的以 autodeploy 开头的分支,
如果该分支没有修改或者修改时间大于一定时间则放弃发布;

2.  根据该分支的修改文件路径判断所属的 产品线/[内网|外网]/机房, 如果
同时有两个 产品线/[内网|外网]/机房, 则也放弃发布;

3. 根据 产品线/[内网|外网]/机房, 生成服务管理系统中对应的 path,
根据 path 去服务管理系统拿到机器列表;

4. 调用发布系统的 API 进行发布, 传入 path, 分支和机器列表.

"""

import os
import sys
import time
import logging
import subprocess

import requests
import ujson as json


logging.basicConfig(
    level=logging.DEBUG, stream=sys.stdout, format='%(message)s')


def shell(cmd, _exit=True):
    """ 执行命令.

    """
    process = subprocess.Popen(
        args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    std_out, std_err = process.communicate()
    return_code = process.poll()
    logging.info("cmd:%s, return_code:%s, std_out:%s, std_err:%s" % (
                  cmd, return_code, std_out, std_err) )
    if return_code == 0:
        return std_out.strip()
    if _exit:
        sys.exit(1)
    return False


class DeployNginxConf(object):
    """ 发布 Nginx conf.

    """
    def __init__(self, git_url, git_dir, shell):
        self.git_url = git_url
        self.git_dir = git_dir
        self.shell = shell

        if not os.path.exists(self.git_dir):
            base_dir = os.path.dirname(self.git_dir.rstrip("/"))
            cmd = "cd %s &&git clone %s" % (base_dir, self.git_url)
            self.shell(cmd)

    def _exe_nginx_conf_cmd(self, cmd):
        """ 在 Nginx conf 目录中执行命令.

        """
        _cmd = "cd %s &&%s" % (self.git_dir, cmd)
        return self.shell(_cmd)

    def get_time_from_string(self, string):
        """ 时间转换.

        格式: Mon Jun 8 13:35:02 2015 ===> 1433741702.0

        """
        time_struct = time.strptime(string, "%a %b %d %H:%M:%S %Y")
        return time.mktime(time_struct)

    def get_branches_from_git(self):
        """ 拿到以 autodeploy 开头的分支列表.

        """
        cmd = "git fetch origin -p &&git branch -r |grep origin/autodeploy"
        ret = self._exe_nginx_conf_cmd(cmd)
        return [ i.strip().replace("origin/", "") for i in ret.strip().splitlines() ]

    def get_lastest_time_for_branch(self, branch):
        """ 拿到一个分支的修改时间.

        这里所有的分支必须从 master 拉取并 push, 命令:
        git checkout master
        git checkout -b autodeployXXX
        git push origin autodeployXXX

        此时分支的修改时间也就是 checkout 时 master 的最后修改时间.

        """
        cmd = "git checkout master &&"\
              "git pull &&"\
              "git checkout %s &&"\
              "git pull " % branch
        self._exe_nginx_conf_cmd(cmd)
        cmd = "git show --date=local --summary `git merge-base %s master`" % branch
        ret = self._exe_nginx_conf_cmd(cmd)
        for i in ret.strip().splitlines():
            if "Date:" in i:
                timestamp = i.replace("Date:", "").strip()
                break
        return self.get_time_from_string(timestamp)

    def get_commitid_from_branch(self):
        """ 拿到分支的 commit id.

        """
        cmd = "git merge-base %s master" % self.branch
        return self._exe_nginx_conf_cmd(cmd)

    def get_changed_file_from_commitid(self, commit_id):
        """ 拿到一个 commit id 修改的文件列表(包括删除).

        """
        cmd = "git diff-tree --no-commit-id --name-only -r %s" % commit_id
        ret = self._exe_nginx_conf_cmd(cmd)
        return [ i.strip() for i in ret.splitlines() ]

    def get_path_from_git(self, _dir):
        """ 根据 git change 的文件路径得到 loki path.

        比如修改的文件路径是: sites-available/public/external/hy/

        则返回结果: /nosa/public/nginx/external/hy

        """
        ret = _dir.strip().split("/")
        return "/nosa/%s/nginx/%s/%s" % (ret[1], ret[2], ret[3])

    def get_hostnames_from_loki(self, path):
        """ 根据 path 从服务管理系统获取主机名列表.

        """
        url = "http://loki.internal.nosa.me/server/api/servers?"\
              "type=path&path=%s" % path
        ret = requests.get(url)
        return [ i["hostname"] for i in ret.json()["data"] ]

    def deploy_conf_from_loki(self):
        """ 调用 loki API 发布配置.

        根据 path, hostnames, branch 三个参数进行发布.

        """
        pass


def main():
    git_url = "ssh://nosa@git.nosa.me:29418/nginx_onf.git"
    git_dir = "/home/jenkins/nginx_conf"

    # 初始化对象.
    deploy_oj = DeployNginxConf(git_url, git_dir, shell)

    # 拿到分支列表.
    branches = deploy_oj.get_branches_from_git()
    # print branches

    # 拿到每个分支的时间.
    times = dict()
    for branch in branches:
        times[branch] = deploy_oj.get_lastest_time_for_branch(branch)
    del branch
    # print times

    # 拿到最新的一个分支, 并判断时间是否在一定范围内.
    # 由于分支的最后修改时间是 checkout 分支时 master 的最后修改时间, 所以修改
    # master 之后需要在一定时间内 checkout 分支并 push, 才能触发此发布任务.
    # 这里给的时间是 600s, 超过 600s 不予发布.
    branch, ctime = sorted(times.items(), key=lambda t:t[1])[-1]
    if time.time() - ctime > 600:
        logging.error("Branch %s is too old" % branch)
        print deploy_oj.__dict__
        sys.exit(1)
    # print branch
    deploy_oj.branch = branch
    del branch

    # 拿到 commit_id, 然后根据 commit_id 拿到改变的文件列表, 然后拿到改变的目录.
    commit_id = deploy_oj.get_commitid_from_branch()
    changed_files = deploy_oj.get_changed_file_from_commitid(commit_id)
    changed_dirs = set()
    for i in changed_files:
        if "sites-available/" in i:
            changed_dirs.add("/".join(i.split("/")[:-1]))
    if len(changed_dirs) != 1:
        logging.error("Too much changed dirs: %s" % changed_dirs)
        sys.exit(1)
    changed_dir = changed_dirs.pop()

    # 根据改名的目录拿到服务管理系统中的 path.
    deploy_oj.path = deploy_oj.get_path_from_git(changed_dir)

    # 根据 path 拿到主机名列表.
    deploy_oj.hostnames = deploy_oj.get_hostnames_from_loki(deploy_oj.path)

    # print deploy_oj.__dict__
    
    # 发布 Nginx conf.
    deploy_oj.deploy_conf_from_loki()    


if __name__ == "__main__":
    main()