Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@cstockton
Created February 28, 2020 15:24
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save cstockton/7f8f24ff5f6f65f6b1457c95d93c91f7 to your computer and use it in GitHub Desktop.
Save cstockton/7f8f24ff5f6f65f6b1457c95d93c91f7 to your computer and use it in GitHub Desktop.
####
# Machine GoBGP configuration: gobgp01
##
[global.config]
as = 26496
router-id = "10.20.30.21"
####
# Environment GoBGP configuration: docker
##
# Neighbors
# Note description is in the form:
#
# region:location:site:fqdn
#
[[neighbors]]
[neighbors.config]
peer-group = "phx"
neighbor-address = "10.20.30.30"
description = "us-west:phx:phx3:ibrma1205-01.phx3.localhost"
[[neighbors]]
[neighbors.config]
peer-group = "phx"
neighbor-address = "10.20.30.31"
description = "us-west:phx:phx3:ibrma1205-02.phx3.localhost"
[[neighbors]]
[neighbors.config]
peer-group = "ash"
neighbor-address = "10.20.30.32"
description = "us-east:ash:ash1:ibrsb0105-01.ash1.localhost"
[[neighbors]]
[neighbors.config]
peer-group = "ams"
neighbor-address = "10.20.30.33"
description = "emea:ams:ams3:ibrsa0105-01.ams3.localhost"
[[neighbors]]
[neighbors.config]
peer-group = "ams"
neighbor-address = "10.20.30.34"
description = "emea:ams:ams3:ibrsa0105-02.ams3.localhost"
# Peer groups
[[peer-groups]]
[peer-groups.config]
peer-group-name = "phx"
peer-as = 26496
[[peer-groups]]
[peer-groups.config]
peer-group-name = "ash"
peer-as = 26496
[[peer-groups]]
[peer-groups.config]
peer-group-name = "ams"
peer-as = 26496
# Mappings
#
# These defined sets together form a mapping which constructs the state
# needed by the protect nodes to orchestrate swings.
[[defined-sets.neighbor-sets]]
neighbor-set-name = "ns-all"
neighbor-info-list = [
"10.20.30.30", "10.20.30.31",
"10.20.30.32",
"10.20.30.33", "10.20.30.34"]
# Peer group to peer address mapping
[[defined-sets.neighbor-sets]]
neighbor-set-name = "ns-pg-phx"
neighbor-info-list = ["10.20.30.30", "10.20.30.31"]
[[defined-sets.neighbor-sets]]
neighbor-set-name = "ns-pg-ash"
neighbor-info-list = ["10.20.30.32"]
[[defined-sets.neighbor-sets]]
neighbor-set-name = "ns-pg-ams"
neighbor-info-list = ["10.20.30.33", "10.20.30.34"]
# System control
#
# Unmanaged means it will be exported to all peers but not be managed by the
# protect system, allowing manual path creation for testing or other purposes.
# It will be removed on export, leaving all other communities.
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-unmanaged"
community-list = ["1:40005"] # 0x19c45 - 105541 - Unmanaged
# Allowed private bgp AS.
[[defined-sets.bgp-defined-sets.as-path-sets]]
as-path-set-name = "as-all"
as-path-list = ["_26496_"]
# Swing community for neustar, used to block reflected swings.
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-neustar"
community-list = ["301:1101"]
# Mitigations
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-bhsrc"
community-list = ["1:40010"] # 0x19c4a - 105546 - Blackhole Src
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-bhdst"
community-list = ["1:40011"] # 0x19c4b - 105547 - Blackhole Dst
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-swing"
community-list = ["1:40012"] # 0x19c4c - 105548 - Traffic Swing
# Swings by peer group
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-swing-phx"
community-list = ["1:40012", "1:40013"]
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-swing-ash"
community-list = ["1:40012", "1:40014"]
[[defined-sets.bgp-defined-sets.community-sets]]
community-set-name = "cs-mitigation-swing-ams"
community-list = ["1:40012", "1:40015"]
####
# Policiy Configuration
##
# Overview:
#
# Peer ->
# adj-In ->
# in-policy: (Route server only)
# (match neighbor-ip IN [neighbor-ip1, neighbor-ip2, ...])
# action: add community in-community
# import-policy:
# (match AS _26496_) AND (match neighbor-ip-list)
# (lookup neighbor-ip in MAP[neighbor-ip]neighbor-community)
# action: Accept route
# action: add community neighbor-community
# loc-rib ->
# export-policy:
# (lookup path community in MAP[neighbor-community]neighbor-ip)
# action: Accept route
# action remove community neighbor-community
# action remove community in-community
# adj-out ->
# (destination)
#
[global.apply-policy.config]
export-policy-list = ["policy-export"]
import-policy-list = ["policy-import"]
default-import-policy = "accept-route"
default-export-policy = "reject-route"
# Export policy - accepts routes that match the destination neighbor community
# to both filter the outbound routes and remove the local community we used for
# on import.
[[policy-definitions]]
name = "policy-export"
# system control: unmanaged path
[[policy-definitions.statements]]
name = "stmt-mitigation-unmanaged"
[policy-definitions.statements.conditions.bgp-conditions.match-community-set]
community-set = "cs-mitigation-unmanaged"
match-set-options = "any"
[policy-definitions.statements.conditions.match-neighbor-set]
neighbor-set = "ns-all"
match-set-options = "any"
[policy-definitions.statements.actions]
route-disposition = "accept-route"
[policy-definitions.statements.actions.bgp-actions.set-community]
options = "remove"
[policy-definitions.statements.actions.bgp-actions.set-community.set-community-method]
communities-list = ["1:40005"]
# mitigation: blackhole src
[[policy-definitions.statements]]
name = "stmt-mitigation-blackhole"
[policy-definitions.statements.conditions.bgp-conditions.match-community-set]
community-set = "cs-mitigation-bhsrc"
match-set-options = "any"
[policy-definitions.statements.conditions.match-neighbor-set]
neighbor-set = "ns-all"
match-set-options = "any"
[policy-definitions.statements.actions]
route-disposition = "accept-route"
[policy-definitions.statements.actions.bgp-actions]
set-next-hop = "192.168.0.1"
[policy-definitions.statements.actions.bgp-actions.set-community]
options = "replace"
[policy-definitions.statements.actions.bgp-actions.set-community.set-community-method]
communities-list = [
"1:0", "200:7707", "26496:60606", "no-export", "no-advertise"]
# mitigation: phx - traffic swing
[[policy-definitions.statements]]
name = "stmt-mitigation-swing-phx"
[policy-definitions.statements.conditions.bgp-conditions.match-community-set]
community-set = "cs-mitigation-swing-phx"
match-set-options = "all"
[policy-definitions.statements.conditions.match-neighbor-set]
neighbor-set = "ns-pg-phx"
match-set-options = "any"
[policy-definitions.statements.actions]
route-disposition = "accept-route"
[policy-definitions.statements.actions.bgp-actions.set-community]
options = "add"
[policy-definitions.statements.actions.bgp-actions.set-community.set-community-method]
communities-list = ["26496:51009"]
# mitigation: ash - traffic swing
[[policy-definitions.statements]]
name = "stmt-mitigation-swing-ash"
[policy-definitions.statements.conditions.bgp-conditions.match-community-set]
community-set = "cs-mitigation-swing-ash"
match-set-options = "all"
[policy-definitions.statements.conditions.match-neighbor-set]
neighbor-set = "ns-pg-ash"
match-set-options = "any"
[policy-definitions.statements.actions]
route-disposition = "accept-route"
[policy-definitions.statements.actions.bgp-actions.set-community]
options = "add"
[policy-definitions.statements.actions.bgp-actions.set-community.set-community-method]
communities-list = ["26496:51009"]
# mitigation: ams - traffic swing
[[policy-definitions.statements]]
name = "stmt-mitigation-swing-ams"
[policy-definitions.statements.conditions.bgp-conditions.match-community-set]
community-set = "cs-mitigation-swing-ams"
match-set-options = "all"
[policy-definitions.statements.conditions.match-neighbor-set]
neighbor-set = "ns-pg-ams"
match-set-options = "any"
[policy-definitions.statements.actions]
route-disposition = "accept-route"
[policy-definitions.statements.actions.bgp-actions.set-community]
options = "add"
[policy-definitions.statements.actions.bgp-actions.set-community.set-community-method]
communities-list = ["26496:51009"]
# Import policy - reject all imports, the GoBGP grpc api is used to check each
# peers adj-in table.
[[policy-definitions]]
name = "policy-import"
# system control: reject all paths from peers.
[[policy-definitions.statements]]
name = "stmt-ibrs-reject"
[policy-definitions.statements.conditions.match-neighbor-set]
neighbor-set = "ns-all"
match-set-options = "any"
[policy-definitions.statements.actions]
route-disposition = "reject-route"

Hello all, anyone have a suggestion for how to achieve the effect of directly injecting a path to a specific peers adj-out? For example given the following setup:

  • GoBGP server has N peer groups of same AS named (e.g. "ams", "iad", "phx")

  • Each peer group has N neighbors, usually 2 (e.g. "ams01", "ams02")

  • Each peer group has a distinct next hop for a given prefix, example adj-in:

    • ams01 adj-in: 50.8.0.0/16 10.20.30.30 ...communities...
    • ams02 adj-in: 50.8.0.0/16 10.20.30.30 ...communities...
    • iad01 adj-in: 50.8.0.0/16 10.20.30.31 ...communities...
    • iad02 adj-in: 50.8.0.0/16 10.20.30.31 ...communities...
    • phx01 adj-in: 50.8.0.0/16 10.20.30.32 ...communities...
    • phx02 adj-in: 50.8.0.0/16 10.20.30.32 ...communities...

Given this, use the GoBGP API to advertise a more specific prefix with the same next hop and one or more new communities, resulting in the following adj-out:

  • ams01 adj-out: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1...
  • ams02 adj-out: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1...
  • iad01 adj-out: 50.8.0.0/24 10.20.30.31 ...communities... ...newcommunities2...
  • iad02 adj-out: 50.8.0.0/24 10.20.30.31 ...communities... ...newcommunities2...
  • phx01 adj-out: 50.8.0.0/24 10.20.30.32 ...communities... ...newcommunities3...
  • phx02 adj-out: 50.8.0.0/24 10.20.30.32 ...communities... ...newcommunities3...

First I tried simply advertising each path into the global rib with the same prefix but different communities which were matched to each peer group in the export policy. But I realized that each advertisement would cause an implicit withdraw of the previous path. I tried setting NoImplicitWithdraw = true, and thought it was exactly what I needed as the global rib contained the 3 distinct paths each with a separate set of communities to match each peer groups export policy, i.e. global rib had:

  • global rib: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1...
  • global rib: 50.8.0.0/24 10.20.30.31 ...communities... ...newcommunities2...
  • global rib: 50.8.0.0/24 10.20.30.32 ...communities... ...newcommunities3...

But each time I added a new route, it would withdraw from the adj-out of the peer matching the specific set of communities. For example given an empty global rib, then advertising a single advertisement of:

  • global rib: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1... ^ last advertisement

I would get the adj-out I wanted:

  • ams01 adj-out: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1...
  • ams02 adj-out: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1...

But as soon as I sent the next prefix to the gobal rib:

  • global rib: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1...
  • global rib: 50.8.0.0/24 10.20.30.31 ...communities... ...newcommunities2... ^ last advertisement

The adj-out for the previous paths would withdraw, leaving a new adj-out of:

  • ams01 adj-out: 50.8.0.0/24 10.20.30.30 ...(MISSING: here to show desired result)
  • ams02 adj-out: 50.8.0.0/24 10.20.30.30 ...(MISSING: here to show desired result)
  • iad01 adj-out: 50.8.0.0/24 10.20.30.31 ...communities... ...newcommunities2...
  • iad02 adj-out: 50.8.0.0/24 10.20.30.31 ...communities... ...newcommunities2...

Finally the last route would be the route that was exported to a neighbor:

  • global rib: 50.8.0.0/24 10.20.30.30 ...communities... ...newcommunities1...
  • global rib: 50.8.0.0/24 10.20.30.31 ...communities... ...newcommunities2...
  • global rib: 50.8.0.0/24 10.20.30.32 ...communities... ...newcommunities3... ^ last advertisement

The adj-out for the previous paths would withdraw, leaving a new adj-out of:

  • ams01 adj-out: 50.8.0.0/24 10.20.30.30 ...(MISSING: here to show desired result)
  • ams02 adj-out: 50.8.0.0/24 10.20.30.30 ...(MISSING: here to show desired result)
  • iad01 adj-out: 50.8.0.0/24 10.20.30.31 ...(MISSING: here to show desired result)
  • iad02 adj-out: 50.8.0.0/24 10.20.30.31 ...(MISSING: here to show desired result)
  • phx01 adj-out: 50.8.0.0/24 10.20.30.32 ...communities... ...newcommunities3...
  • phx02 adj-out: 50.8.0.0/24 10.20.30.32 ...communities... ...newcommunities3...

Side note: The behavior above doesn't seem to send a WITHDRAW event when monitoring the rib. So to my local route caches it looked like the route was being accepted to each peer, since the previous route was never withdrawn from my local cache. It wasn't until I checked the actual peers routing tables directly that I noticed the route missing.

Since then I have tried many types of configurations and work arounds. Trying a route server mode looked promising because it seemed that each neighbor had its own distinct rib. But I can't find a way to inject directly into the local rib of a specific neighbor using the GRPC API as there is no "Name" param like the corresponding GetRib / GetPath requests.

For exmaple when selecting the next hop for the advertisements above I use the neighbors address as the "Name" in the GetRib / GetPath requests with a resource of Resource_ADJ_IN. However AddPath has no way to specify a Resource_ADJ_OUT (which sort of makes since, it would circumvent route policy, but also be very useful I think for strictly managed routing via API).

I have also tried VRF since it provides distinct routing tables, but I am not familiar with RT / RD / import / export aspects of it so getting a working POC was a struggle. Not knowing if it was the right tool for my problem made me punt exploring further.

Finally I've tried various combinations of local / remote AS since changing the peer AS is an option. But I can't get paths to export at all in these cases, my BGP knowledge may be the limiting factor here but I struggled to actually set the source AS through the table API. I ended up crashing the GoBGP server no matter how I attempted to add a bgp.NewAsPathAttribute. When using NewAsPath4.. it crashed with:

"Topic":"Table","msg":"AS4_PATH must be converted to AS_PATH"

But when using NewAsPath it crashed with:

"Topic":"Table","msg":"AsPathParam must be converted to As4PathParam"

This is probably a gap in my BGP knowledge / path construction, my question is could the use case be solved with the right API usage and replacing community driven export policy with an "as-path" matching one?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment