README.md

TerraChecks

Скрипты для сравнения состояния OpenStack-облака Селектела, записей DNSimple и соответствующих файлов состояния терраформа.

Usage

1. Добавить в корень этот проекта файл secret.yml:

DNSIMPLE_LOGIN: ...
DNSIMPLE_PASSWORD: ...
SELECTEL_KEY: ...
GITHUB_TOKEN: ...
GITHUB_ORG_NAME: ...

2. Запустить скрипт ./backup

3. PROFIT! Полученные папки стоит хранить в приватном репозитарии для истории.

Hack Points

1. ./backup_terrform.rb REPOS_RE - регексп, который описывает в каких репозитариях лежат файлы состояния, чтобы не клонировать все репозитарии.
2. ./check.rb check_terraform_dns - на 11-й строчке есть место, где можно проигнорировать записи, которые настраиваются не через терраформ. Dnsimple-провайдер поддерживает не все типы записей, ну и некоторые экзотические штуки иногда удобнее добавить вручную.

_secret.rb

require 'yaml'
require 'json'

SECRET_KEYS = %w[
  DNSIMPLE_LOGIN
  DNSIMPLE_PASSWORD
  SELECTEL_KEY
  GITHUB_TOKEN
  GITHUB_ORG_NAME
]

def get_secret!
  yaml = File.exists?("secret.yml") ?  YAML.load(File.read("secret.yml")) : {}

  if (SECRET_KEYS - yaml.keys).length > 0
    raise "missed #{(SECRET_KEYS - yaml.keys).inspect} keys in secret.yml"
  end

  yaml
end

backup

#!/bin/bash

time (
  ./backup_dnsimple.rb &&
  ./backup_selectel.rb &&
  ./backup_terrform.rb &&
  echo "" &&
  echo "" &&
  ./check.rb &&
  echo DONE
)

backup_dnsimple.rb

#!/usr/bin/env ruby
require './_secret.rb'
secret = get_secret!

def dns(secret, url)
  JSON.load `curl -q -u #{secret["DNSIMPLE_LOGIN"]}:#{secret["DNSIMPLE_PASSWORD"]} -H 'Accept: application/json' https://api.dnsimple.com/v1#{url}`
end

system "rm -fr domains; mkdir -p domains"
dns(secret, "/domains").each do |domain|
  name = domain["domain"]["name"]
  data = dns(secret, "/domains/#{name}/records")
  File.write("domains/#{name}.txt", data.sort_by { |a| 
    a["record"]["id"]
  }.map { |a|
    [
      a["record"]["id"],
      a["record"]["parent_id"],
      a["record"]["record_type"],
      a["record"]["system_record"],
      a["record"]["name"],
      a["record"]["ttl"],
      a["record"]["prio"],
      a["record"]["content"],
    ].join(" -- ")
  }.join("\n") + "\n")
  puts "#{name} backuped"
end

puts "Done."

backup_selectel.rb

#!/usr/bin/env ruby
require './_secret.rb'
secret = get_secret!

out1 = send :`, %(curl -H 'X-token: #{secret["SELECTEL_KEY"]}' 
  https://api.selectel.ru/vpc/resell/v2/projects).gsub("\n", " ")

system "rm -fr selectel; mkdir -p selectel"

JSON.load(out1)["projects"].each do |project|
  puts "_____ PROJECT: #{project["name"]} _______"
  out2 = send :`, %(curl -H 'X-token: #{secret["SELECTEL_KEY"]}' 
    -X POST 
    -H 'Content-Type: application/json'
    -d '{"token":{"project_id":"#{project["id"]}"}}'
    https://api.selectel.ru/vpc/resell/v2/tokens).gsub("\n", " ")
  token = JSON.load(out2)["token"]["id"]

  out3 = send :`, %(curl
    -X POST
    -H 'Content-Type: application/json'
    -d '{"auth":{
          "identity":{
            "methods":["token"],
            "token": {
              "id": "#{token}"
            }
          },
          "scope": {
            "project": {
              "id": "#{project["id"]}"
            }
          }
    }}'
    https://api.selvpc.ru/identity/v3/auth/tokens
  ).gsub("\n", " ")

  volumes = {
  }
  detached_volumes = []
  JSON.load(out3)["token"]["catalog"].find{ |item| 
    item["name"] == "cinder" && item["type"] == "volumev2"
  }["endpoints"].select { |ep|
    ep["interface"] == "public"
  }.each do |ep|
    out6 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/volumes)
    JSON.load(out6)["volumes"].each do |volume|
      out7 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/volumes/#{volume["id"]})
      v = JSON.load(out7)["volume"]
      if v["attachments"].length >= 1
        a = v["attachments"][0]
        volumes[a["server_id"]] ||= []
        volumes[a["server_id"]].push({
          "id" => v["id"],
          "name" => volume["name"],
          "size" => v["size"],
          "type" => v["volume_type"],
          "device" => a["device"]
        })
      else 
        detached_volumes.push(
          "id" => v["id"],
          "region" => ep["region"],
          "name" => volume["name"],
          "size" => v["size"],
          "type" => v["volume_type"]
        )
      end
    end
  end

  servers = []
  JSON.load(out3)["token"]["catalog"].find{ |item| 
    item["name"] == "nova" && item["type"] == "compute"
  }["endpoints"].select { |ep|
    ep["interface"] == "public"
  }.each do |ep|
    out4 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/servers)
    JSON.load(out4)["servers"].each do |server|
      out5 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/servers/#{server["id"]})
      s = JSON.load(out5)["server"]
      out8 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/flavors/#{s["flavor"]["id"]})
      flavor = JSON.load(out8)["flavor"]
      servers.push(
        "region" => ep["region"],
        "name" => server["name"],
        "id" => server["id"],
        "ram" => flavor["ram"],
        "cpus" => flavor["vcpus"],
        "ips" => s["addresses"].values.flatten.map{ |a| a["addr"] }.join(", ")
      )
    end
  end

  networks = []
  JSON.load(out3)["token"]["catalog"].flat_map { |item| item["endpoints"] }.select { |ep|
    ep["interface"] == "public" && ep["name"] == "network"
  }.each do |ep|
    out4 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/v2.0/networks)
    JSON.load(out4)["networks"].reject { |network| 
      # пропустить внешнюю сеть и публичные
      network["name"] == "external-network" || network["name"] =~ /\/29$/
    }.each do |network|
      networks.push(
        "region" => ep["region"],
        "name" => network["name"],
        "id" => network["id"]
      )
    end
  end

  subnets = []
  JSON.load(out3)["token"]["catalog"].flat_map { |item| item["endpoints"] }.select { |ep|
    ep["interface"] == "public" && ep["name"] == "network"
  }.each do |ep|
    out4 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/v2.0/subnets)
    JSON.load(out4)["subnets"].reject { |x|
      # пропустить публичные
      x["name"] =~ /\/29$/
    }.each do |x|
      subnets.push(
        "region" => ep["region"],
        "name" => x["name"],
        "id" => x["id"]
      )
    end
  end

  routers = []
  JSON.load(out3)["token"]["catalog"].flat_map { |item| item["endpoints"] }.select { |ep|
    ep["interface"] == "public" && ep["name"] == "network"
  }.each do |ep|
    out4 = send :`, %(curl -H 'X-Auth-Token: #{token}' #{ep["url"]}/v2.0/routers)
    JSON.load(out4)["routers"].each do |x|
      routers.push(
        "region" => ep["region"],
        "name" => x["name"],
        "id" => x["id"]
      )
    end
  end

  File.open("selectel/#{project["name"]}.txt", "w") do |f|
    detached_volumes.sort_by { |v| v["name"] }.each do |v|
      f.puts "#{v["region"]}: #{v["name"]} [#{v["size"]} MB #{v["type"]}] - (#{v["id"]})"
    end
    if detached_volumes.length > 0
      f.puts ""
      f.puts ""
    end

    servers.sort_by { |s| s["name"] }.each do |s|
      f.puts "#{s["region"]}: #{s["name"]} (#{s["id"]}) - #{s["ram"]} MB, #{s["cpus"]} CPU, IPs: #{s["ips"]}"
      (volumes[s["id"]] || {}).sort_by { |v| v["device"] }.each do |v|
        f.puts "  #{v["device"]} == #{v["name"]} [#{v["size"]} MB #{v["type"]}] - (#{v["id"]})"
      end
      f.puts ""
    end

    f.puts "Networks:"
    networks.sort_by { |n| n["name"] }.each do |n|
      f.puts "  #{n["region"]}: #{n["name"]} (#{n["id"]})"
    end

    f.puts "Subnets:"
    subnets.sort_by { |n| n["name"] }.each do |n|
      f.puts "  #{n["region"]}: #{n["name"]} (#{n["id"]})"
    end

    f.puts "Routers:"
    routers.sort_by { |n| n["name"] }.each do |n|
      f.puts "  #{n["region"]}: #{n["name"]} (#{n["id"]})"
    end
  end
end

puts "Done."

backup_terrform.rb

#!/usr/bin/env ruby
require './_secret.rb'
secret = get_secret!

TOKEN = secret["GITHUB_TOKEN"]
ORG_NAME = secret["GITHUB_ORG_NAME"]

# В каких репозитариях есть терраформ файлы
REPOS_RE = /^ansible-.*$/
# Путь до файлов состояния
TF_STATES = "**/terraform.tfstate"

def get_paginate(cmd)
  url = "https://api.github.com#{cmd}"
  out = []

  while url
    out += JSON.parse(`curl -H "Authorization: token #{TOKEN}" #{url}`)
    headers = `curl -I -H "Authorization: token #{TOKEN}" #{url}`.strip.split("\n").map { |h|
      h.strip.split(": ", 2) unless h =~ /^HTTP/
    }.compact.to_h

    url = nil
    if headers["Link"]
      # puts "[debug] headers[Link] = #{headers["Link"]}"
      headers["Link"].split(", ").each do |part|
        a, b = part.split("; ", 2)
        if b == 'rel="next"'
          url = a[1..-2]
        end
      end
    end
  end

  out
end

git_repos = get_paginate("/orgs/#{ORG_NAME}/repos").map { |repo| repo["html_url"] }.sort.select { |r|
  File.basename(r) =~ REPOS_RE
}

system "mkdir -p .repos"
(Dir[".repos/*"].map { |r| r.sub(".repos/", "") } - git_repos.map { |r| File.basename(r) }).each do |dir|
  puts "rm -fr .repos/#{dir}"
  system "rm -fr .repos/#{dir}"
end
git_repos.map { |r| File.basename(r) }.each do |repo|
  if File.exists?(".repos/#{repo}")
    system "cd .repos/#{repo} && git pull"
  else
    system "cd .repos && git clone git@github.com:#{ORG_NAME}/#{repo}"
  end
end

system "rm -fr terraform"
system "mkdir -p terraform"
state = Dir[".repos/#{TF_STATES}"].flat_map { |tf|
  JSON.load(File.read(tf))["modules"].flat_map { |m| 
    m["resources"].values
  }
}
File.write("terraform/state.json", JSON.pretty_generate(state) + "\n")

puts "Done."

check.rb

#!/usr/bin/env ruby
require 'json'
require 'time'
require 'digest/md5'

def check_terraform_dns
  return nil
  Dir["domains/*.txt"].each do |file|
    domain = File.basename(file, ".txt")
    dns = File.readlines(file).reject { |line|
      # <-- insert here some reject rules
      false
    }.map { |line|
      [line.split(" ").first.to_i, line.strip]
    }.to_h

    tf = JSON.load(File.read("terraform/state.json")).select { |x|
      x["type"] == "dnsimple_record" && x["primary"]["attributes"]["domain"] == domain
    }.map { |x| 
      [x["primary"]["id"].to_i, x]
    }.to_h

    if (tf.keys - dns.keys).count > 0
      return "[#{domain}] Terraform has unknown keys:\n" + (tf.keys - dns.keys).map { |k| tf[k].inspect }.join("\n")
    end

    if (dns.keys - tf.keys).count > 0
      return "[#{domain}] DNS has unknown keys:\n" + (dns.keys - tf.keys).map { |k| dns[k] }.join("\n")
    end
  end
  nil
end

def check_terraform_selectel
  re = /\((\h+-\h+-\h+-\h+-\h+)\)/
  selectel = Dir["selectel/*.txt"].flat_map { |file| 
    File.readlines(file).map { |l| File.basename(file) + ": " + l.strip }
  }.select { |line|
    line =~ re
  }.map { |line|
    line =~ re
    [$1, line]
  }.to_h

  tf = JSON.load(File.read("terraform/state.json")).select { |x|
    %w[
      openstack_compute_instance_v2
      openstack_blockstorage_volume_v1
      openstack_blockstorage_volume_v2
      openstack_networking_network_v2
      openstack_networking_subnet_v2
      openstack_networking_router_v2
    ].index(x["type"])
  }.map { |x| 
    [x["primary"]["id"], x]
  }.to_h

  if (tf.keys - selectel.keys).count > 0
    return "Terraform has unknown keys:\n" + (tf.keys - selectel.keys).map { |k| tf[k].inspect }.join("\n")
  end

  if (selectel.keys - tf.keys).count > 0
    return "Selectel has unknown keys:\n" + (selectel.keys - tf.keys).map { |k| selectel[k] }.join("\n")
  end
end


[
  "check_terraform_dns",
  "check_terraform_selectel",
].each_with_index.map { |check, index|
  if ARGV[0].to_i > 0 && ARGV[0].to_i != index + 1
    {check: check, error: "skipped"}
  else
    {check: check, error: send(check)}
  end
}.each_with_index do |result, index|
  next if ARGV[0].to_i > 0 && ARGV[0].to_i != index + 1
  out = ("%05s. " % (index+1)) + result[:check]
  if result[:error]
    out += " [\e[31mFAIL\e[0m] - #{result[:error]}"
  else
    out += " [\e[32mSUCC\e[0m]"
  end
  puts out
end