Skip to content

Instantly share code, notes, and snippets.

@thearchitect
Created December 10, 2016 17:00
Show Gist options
  • Save thearchitect/959f3c05c486bbb4e35244e800249ad1 to your computer and use it in GitHub Desktop.
Save thearchitect/959f3c05c486bbb4e35244e800249ad1 to your computer and use it in GitHub Desktop.
{ config, lib, pkgs, ... }:
let
cfg = {
hostName = "zion";
ceph = {
enable = true;
publicIP = "127.0.0.1";
privateIP = "127.0.0.1";
osds = [
{ id = 1; monID = "a"; uuid = "342e8fed-d2b9-4c50-aef4-0cba692c1aeb"; dev = "sda"; }
{ id = 2; monID = "b"; uuid = "e8c0eeb1-4161-4bab-ad4c-ed82325982fe"; dev = "sdb"; }
{ id = 3; monID = "c"; uuid = "519995d7-9d38-4b65-ab8b-e7162b0c27bf"; dev = "sdc"; }
{ id = 4; monID = "d"; uuid = "2537b082-d36d-4e64-aaea-c40ee04bcebe"; dev = "sdd"; }
];
};
};
in
{
system.stateVersion = "16.09";
imports =
[
<nixpkgs/nixos/modules/installer/scan/not-detected.nix>
];
boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" "sdhci_pci" ];
boot.kernelModules = [
"pcspkr"
"usb_storage"
"tun"
"virtio"
"kvm" "kvm-intel"
"ceph" "rbd"
"btrfs"
"nvidia" "nvidia_uvm" "nvidia_drm" "nvidia_modeset"
];
boot.extraModulePackages = [ ];
fileSystems = {
"/" = {
device = "/dev/disk/by-uuid/1a381f95-4614-42c9-9b04-751af97aabf7";
# device = "/dev/disk/by-label/nixos-usb";
fsType = "btrfs";
options = [ "subvol=nixos" "ssd" "autodefrag" "noatime" "nospace_cache" "compress-force=zlib" ];
};
"/boot/efi" = {
device = "/dev/disk/by-uuid/0FF6-D04F";
options = [ "noatime" ];
};
"/tmp" = {
device = "tmpfs";
fsType = "tmpfs";
options = [
"noexec"
"nodev"
"nosuid"
"size=8192m"
"mode=1777"
];
};
};
swapDevices = [];
nix.maxJobs = lib.mkDefault 1;
boot.loader.grub.enable = true;
boot.loader.grub.version = 2;
boot.loader.grub.efiSupport = true;
boot.loader.grub.efiInstallAsRemovable = true;
boot.loader.efi.efiSysMountPoint = "/boot/efi";
boot.loader.grub.device = "nodev"; # or "nodev" for efi only
boot.loader.timeout = 1;
boot.kernelPackages = pkgs.linuxPackages_4_8;
networking = {
hostName = cfg.hostName;
nameservers = [ "127.0.0.1" "8.8.8.8" ];
firewall = {
enable = true;
allowPing = true;
trustedInterfaces = [ "wlan0_ap" "lan0" "lan1" ];
checkReversePath = false;
allowedTCPPorts = [
22 # ssh
80 # http
443 # https
];
allowedUDPPorts = [ ];
};
nat = {
enable = true;
internalIPs = [ "10.0.3.0/24" "10.0.4.0/24" "10.0.5.0/24" ];
externalInterface = "wan1_modem_yota";
};
interfaces = {
wlan0_ap = {
ipAddress = "10.0.3.1";
prefixLength = 24;
ip4 = [ { address = "10.0.3.1"; prefixLength = 24; } ];
};
lan0 = {
ipAddress = "10.0.4.1";
prefixLength = 24;
ip4 = [ { address = "10.0.4.1"; prefixLength = 24; } ];
};
lan1 = {
ipAddress = "10.0.5.1";
prefixLength = 24;
ip4 = [ { address = "10.0.5.1"; prefixLength = 24; } ];
};
wan1_modem_yota = {
useDHCP = true;
};
};
networkmanager = {
enable = false;
};
};
time.timeZone = "Etc/GMT-4";
nixpkgs = {
config = {
allowUnfree = true;
};
};
environment = {
systemPackages = with pkgs; [
zile #emacs24-nox
mc
htop
wget
ceph
kvm
#qemu #spice-vdagent
#win-qemu
xen
#networkmanager
iproute
iw wirelesstools rfkill
nssmdns
lsof
hdparm
btrfs-progs
#dropbox-cli
(pkgs.substituteAll {
name = "nix+force-cleanup";
dir = "bin";
isExecutable = true;
src = pkgs.writeScript "nix+force-cleanup" ''#! ${pkgs.stdenv.shell} -e
${pkgs.zsh}/bin/zsh << ZSHEOF
if [[ $UID == 0 || $EUID == 0 ]]; then
${pkgs.nix}/bin/nix-env --delete-generations 2d
${pkgs.nix}/bin/nix-store --gc --print-dead
${pkgs.nix}/bin/nix-store --gc
${pkgs.nix}/bin/nix-collect-garbage -d
${pkgs.nix}/bin/nix-store --optimise
else
echo Please, run this as root
fi
ZSHEOF'';
})
(pkgs.substituteAll {
name = "force-ceph-mkfs";
dir = "bin";
isExecutable = true;
src = pkgs.writeScript "force-ceph-mkfs" ''#! ${pkgs.stdenv.shell}
dev=/dev/$1
mp=/tmp/$1
#${pkgs.coreutils}/bin/dd if=/dev/zero of=$dev bs=8M count=8 status=progress
${pkgs.btrfs-progs}/bin/mkfs.btrfs -f $dev
${pkgs.coreutils}/bin/mkdir $mp
${pkgs.utillinux}/bin/mount $dev $mp
${pkgs.btrfs-progs}/bin/btrfs subvolume create $mp/osd
${pkgs.btrfs-progs}/bin/btrfs subvolume create $mp/mon
${pkgs.btrfs-progs}/bin/btrfs subvolume create $mp/mds
${pkgs.utillinux}/bin/umount $mp
${pkgs.coreutils}/bin/rmdir $mp
'';
})
];
noXlibs = true;
};
programs = {
ssh = {
startAgent = false;
};
bash = {
enableCompletion = true;
};
};
services = {
udev = {
extraRules = ''
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wlan0_ap"
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wlan1_ap"
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wan1_modem_yota"
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="wan0"
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="lan0"
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="", ATTR{dev_id}=="0x0", ATTR{type}=="1", NAME="lan1"
'';
};
xserver.enable = false;
dbus = {
#packages = with pkgs; [ avahi ];
};
openssh = {
enable = true;
permitRootLogin = "without-password";
passwordAuthentication = false;
gatewayPorts = "yes";
extraConfig = ''
PasswordAuthentication=no
KbdInteractiveAuthentication=no
ChallengeResponseAuthentication=no
PermitTunnel yes
'';
};
avahi = {
enable = true;
nssmdns = true;
hostName = "${config.networking.hostName}";
wideArea = true;
ipv4 = true;
ipv6 = true;
publish = {
enable = true;
domain = true;
addresses = true;
hinfo = true;
userServices = true;
workstation = true;
};
# todo:
#[server]
#allow-point-to-point=yes
#deny-interfaces=eth1
#[reflector]
#enable-reflector=yes
};
dnsmasq = {
enable = true;
servers = [ "8.8.8.8" "8.8.4.4" ];
extraConfig = ''
listen-address=127.0.0.1,10.0.3.1,10.0.4.1,10.0.5.1
domain=lan
interface=wlan0_ap
interface=lan0
interface=lan1
bind-interfaces
dhcp-range=10.0.3.16,10.0.3.254,24h
dhcp-range=10.0.4.16,10.0.4.254,24h
dhcp-range=10.0.5.16,10.0.5.254,24h
'';
};
hostapd = {
enable = true;
interface = "wlan0_ap";
ssid = "";
wpa = true;
wpaPassphrase = "";
#channel = 11;
hwMode = "g";
extraConfig = ''
wpa=2
wpa_key_mgmt=WPA-PSK
wpa_pairwise=TKIP
rsn_pairwise=CCMP
auth_algs=3
macaddr_acl=0
ieee80211n=1
#ieee80211ac=1
wmm_enabled=1
'';
};
ntp = {
enable = true;
servers = [
"0.pool.ntp.org"
"1.pool.ntp.org"
"2.pool.ntp.org"
"3.pool.ntp.org"
];
};
gpm = {
enable = false;
protocol = "ps/2";
};
atd = {
enable = true;
allowEveryone = true;
};
udisks2 = {
enable = true;
};
};
virtualisation = {
libvirtd = {
enable = false;
};
};
environment.etc."ceph/ceph.conf" = {
mode = "0644";
text = lib.concatStringsSep "\n" ([''
[global]
fsid =
public network = ${cfg.ceph.publicIP}/24
cluster network = ${cfg.ceph.privateIP}/24
auth cluster required = none
auth service required = none
auth client required = none
log to syslog = false
[mon]
debug mon = 9
mon osd full ratio = .70
mon osd nearfull ratio = .60
mon osd down out interval = 0
mon osd report timeout = 300
[osd]
filestore btrfs snap = true
filestore max sync interval = 5
#osd crush update on start = true
#[mds]
'']
++ (map (osd: ''
[mon.${osd.monID}]
host = ${cfg.hostName}
mon addr = ${cfg.ceph.publicIP}:${toString (6789 + osd.id)}
mon data = /ceph/${osd.dev}/mon
[osd.${toString osd.id}]
host = ${cfg.hostName}
osd data = /ceph/${osd.dev}/osd/data
osd journal = /ceph/${osd.dev}/osd/journal
[mds.${toString osd.id}]
host = ${cfg.hostName}
mds data = /ceph/${osd.dev}/mds
'') (cfg.ceph.osds))
);
};
systemd.services = {
# builtins.listToAttrs (lib.flip map cfg.osds (osd: lib.nameValuePair "ceph-osd-${toString osd.id}" {
} // (builtins.listToAttrs (builtins.concatLists (lib.flip map cfg.ceph.osds (osd: [
{
name = "ceph-mon-${toString osd.monID}";
value = {
################################################################
####
#### Ceph MON
####
enable = cfg.ceph.enable;
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
requires = [ ];
restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ];
path = with pkgs; [ ceph zsh lsb-release getopt ];
script = ''${pkgs.ceph}/bin/ceph-mon -i ${osd.monID} -c /etc/ceph/ceph.conf --public-addr ${cfg.ceph.publicIP}:${toString (6789 + osd.id)} -f -d'';
serviceConfig = {
Type = "simple";
PermissionsStartOnly = true;
Restart = "always";
RestartSec = 1;
StartLimitInterval = 0;
};
preStart = ''
${pkgs.hdparm}/bin/hdparm -W 0 /dev/${osd.dev}
${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mon || true
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mon || true
${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/mon
${pkgs.utillinux}/bin/mount -t btrfs -o rw,sync,noatime,autodefrag,nospace_cache,compress-force=zlib,subvol=mon /dev/${osd.dev} /ceph/${osd.dev}/mon
${pkgs.ceph}/bin/ceph-mon --mkfs -i ${osd.monID} -c /etc/ceph/ceph.conf -f -d
'';
postStop = ''
${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mon
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mon
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
'';
####
#### EOF Ceph MON
####
################################################################
};
}
{
name = "ceph-osd-${toString osd.id}";
value = {
################################################################
####
#### Ceph OSD
####
enable = cfg.ceph.enable;
wantedBy = [ "multi-user.target" ];
after = [ "ceph-mon-${toString osd.monID}.service" ];
requires = [ "ceph-mon-${toString osd.monID}.service" ];
restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ];
path = with pkgs; [ ceph utillinux hdparm ceph zsh lsb-release getopt coreutils btrfs-progs ];
script = ''${pkgs.ceph}/bin/ceph-osd -i ${toString osd.id} -c /etc/ceph/ceph.conf -f -d'';
serviceConfig = {
Type = "simple";
PermissionsStartOnly = true;
Restart = "always";
RestartSec = 1;
StartLimitInterval = 0;
};
preStart = ''${pkgs.zsh}/bin/zsh << EOF
${pkgs.hdparm}/bin/hdparm -W 0 /dev/${osd.dev}
${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/osd || true
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/osd || true
${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/osd
${pkgs.utillinux}/bin/mount -t btrfs -o rw,sync,noatime,autodefrag,nospace_cache,compress-force=zlib,subvol=osd /dev/${osd.dev} /ceph/${osd.dev}/osd
${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/osd/data
${pkgs.ceph}/bin/ceph osd create ${osd.uuid} ${toString osd.id} | true
${pkgs.ceph}/bin/ceph-osd --mkfs -i ${toString osd.id} --osd-uuid ${osd.uuid} -c /etc/ceph/ceph.conf -f -d | true
EOF'';
postStop = ''
${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/osd
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/osd
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
'';
####
#### EOF Ceph OSD
####
################################################################
};
}
{
name = "ceph-mds-${toString osd.id}";
value = {
################################################################
####
#### Ceph MDS
####
enable = cfg.ceph.enable;
wantedBy = [ "multi-user.target" ];
after = [ "ceph-mon-${toString osd.monID}.service" ];
requires = [ "ceph-mon-${toString osd.monID}.service" ];
restartTriggers = [ config.environment.etc."ceph/ceph.conf".source ];
path = with pkgs; [ ceph zsh lsb-release getopt ];
script = ''${pkgs.ceph}/bin/ceph-mds -i ${toString osd.id} -c /etc/ceph/ceph.conf -f -d'';
serviceConfig = {
Type = "simple";
PermissionsStartOnly = true;
Restart = "always";
RestartSec = 1;
StartLimitInterval = 0;
};
preStart = ''
${pkgs.hdparm}/bin/hdparm -W 0 /dev/${osd.dev}
${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mds || true
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mds || true
${pkgs.coreutils}/bin/mkdir -p /ceph/${osd.dev}/mds
${pkgs.utillinux}/bin/mount -t btrfs -o rw,sync,noatime,autodefrag,nospace_cache,compress-force=zlib,subvol=mds /dev/${osd.dev} /ceph/${osd.dev}/mds
'';
postStop = ''
${pkgs.utillinux}/bin/umount /ceph/${osd.dev}/mds
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev}/mds
${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
'';
####
#### EOF Ceph MDS
####
################################################################
};
}
]))));
security = {
sudo = {
enable = true;
wheelNeedsPassword = false;
};
};
users = {
mutableUsers = false;
users = {
keeper = {
isNormalUser = true;
extraGroups = [ "wheel" ];
};
root.openssh.authorizedKeys.keys = [
];
};
};
}
@bjornfor
Copy link

-${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} | true
+${pkgs.coreutils}/bin/rmdir /ceph/${osd.dev} || true

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment