Last active
April 23, 2024 23:27
-
-
Save alexf101/b65cbfe7c5a61df7d925589a71d200cf to your computer and use it in GitHub Desktop.
Translates a Dependabot config with glob support to a real Dependabot config
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env ruby | |
require 'bundler/inline' | |
gemfile do | |
source 'https://rubygems.org' | |
gem 'httparty' | |
gem 'fast_ignore' | |
end | |
require 'httparty' | |
require 'yaml' | |
require 'set' | |
require 'json' | |
require 'pp' | |
CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml' | |
DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml' | |
REPO_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) | |
Dir.chdir REPO_ROOT | |
GLOBAL_EXCLUDE_REGEXES = [ | |
%r{\/node_modules\/}, | |
] | |
# Validates the special config file that understands Stile's extensions. | |
def validate_generator_config(config_name, config) | |
raise 'Missing updates?' unless config['updates'].length >= 1 | |
config['updates'].each do |target| | |
raise 'Missing package-ecosystem' unless target['package-ecosystem'] | |
raise 'Missing schedule' unless target['schedule'] | |
raise 'Missing apply-to-glob' unless target['apply-to-glob'] | |
target | |
.fetch('ignore', []) | |
.each do |ignored_update| | |
dependency_name = ignored_update['dependency-name'] | |
unless dependency_name | |
raise "Missing dependency name: #{target.pretty_inspect}" | |
end | |
unless ignored_update['blocked-reason'] | |
raise "#{dependency_name}: Missing blocked-reason" | |
end | |
# This whitelist of valid values for blocked-reason intentionally makes it a bit difficult to add new ones willy-nilly. Please consider carefully whether whatever the new reason is, is actually a good enough reason to block updates. | |
unless %w[ | |
bad-upstream-version | |
deletion-planned | |
requires-other-project:drop-ie11 | |
requires-other-project:delete-backbone | |
].include?(ignored_update['blocked-reason']) | |
raise "#{dependency_name}: Invalid value for blocked-reason" | |
end | |
unless ignored_update['blocked-reason-detail'] | |
raise "#{dependency_name}: Missing blocked-reason-detail" | |
end | |
end | |
end | |
config | |
rescue StandardError | |
STDERR.puts "Validation failed for input config file at `#{config_name}`" | |
raise | |
end | |
# Validates the generated vanilla Dependabot config files (to let us test that we didn't stuff up the logic) | |
def validate_generated_config(config_name, generated_config) | |
raise 'Missing version' unless generated_config['version'] | |
raise 'Missing updates' unless generated_config['updates'] | |
generated_config['updates'].each do |target| | |
raise 'Missing directory' unless target['directory'] | |
valid_keys = %w[ | |
package-ecosystem | |
directory | |
schedule | |
allow | |
assignees | |
commit-message | |
ignore | |
insecure-external-code-execution | |
labels | |
milestone | |
open-pull-requests-limit | |
pull-request-branch-name | |
rebase-strategy | |
registries | |
reviewers | |
target-branch | |
vendor | |
versioning-strategy | |
] | |
invalid_keys = target.keys - valid_keys | |
if invalid_keys.length > 0 | |
raise "Contains invalid keys: #{invalid_keys}" | |
end | |
target | |
.fetch('ignore', []) | |
.each do |ignored_update| | |
dependency_name = ignored_update['dependency-name'] | |
raise 'Missing dependency name' unless dependency_name | |
invalid_keys = | |
ignored_update.keys - %w[ | |
dependency-name | |
version_requirement | |
] | |
if invalid_keys.length > 0 | |
raise "Contains invalid keys: #{invalid_keys}" | |
end | |
end | |
end | |
generated_config | |
rescue StandardError | |
STDERR.puts "Validation failed for config file `#{config_name}`" | |
raise | |
end | |
# Core logic of this script - translates our extensions into a vanilla Dependabot format through judicious use of glob and regex. | |
def generate_config(config_name, config_generator_config) | |
config = YAML.load(config_generator_config) | |
validate_generator_config(config_name, config) | |
new_config = YAML.load(config_generator_config) | |
new_config['updates'] = [] | |
config['updates'].each do |target| | |
apply_to_glob = target['apply-to-glob'] | |
exclude_regex = target['exclude-regex'] | |
# Find all matching glob files, while applying gitignore rules | |
matched_files = | |
FastIgnore.new( | |
relative: true, | |
argv_rules: [apply_to_glob], | |
root: REPO_ROOT, | |
).to_a.select do |path| | |
!GLOBAL_EXCLUDE_REGEXES.any? do |global_exclude_regex| | |
global_exclude_regex.match?(path) | |
end | |
end.sort | |
if exclude_regex | |
exclude_regex_parsed = Regexp.new(exclude_regex) | |
matched_files = | |
matched_files.reject do |path| | |
exclude_regex_parsed.match?(path) | |
end | |
end | |
if target['ignore'] | |
target['ignore'].each do |ignored_update| | |
ignored_update.reject! do |k| | |
%w[blocked-reason blocked-reason-detail].include?(k) | |
end | |
end | |
end | |
matched_files.each do |filepath| | |
new_config['updates'] << | |
{ | |
# Directory is relative to the repository's root, with '/' indicating the repo's root. However, it also seems to find '.' and relative paths without '/' acceptable, so... let's just do that then. | |
# See https://dependabot.com/docs/config-file/validator/ for confirmation. | |
'directory' => File.dirname(filepath), | |
}.merge( | |
target.reject do |k| | |
%w[apply-to-glob exclude-regex].include?(k) | |
end, | |
) | |
end | |
# Remove duplicates - later entries should override earlier ones. | |
found_so_far = Set.new | |
new_config['updates'] = | |
new_config['updates'].reverse.reject do |target| | |
unique_key = [target['directory'], target['package-ecosystem']] | |
should_skip = found_so_far.include?(unique_key) | |
found_so_far << unique_key | |
should_skip | |
end.sort_by do |target| | |
# Stable output important to minimise human-readable diff. | |
# | |
# There is no "primary key" for entries in the Dependabot | |
# config schema (https://dependabot.com/docs/config-file/) | |
# but it seems unlikely that we'll have multiple entries | |
# that share the same directory and package manager -- e.g. | |
# updating exactly the same stuff but on a different schedule. | |
[target['directory'], target['package-ecosystem']] | |
end | |
end | |
validate_generated_config(config_name, new_config) | |
new_config | |
end | |
### UNIT TESTS ### | |
# May as well run them every time the script runs for a wee little project like this. | |
cfg = generate_config('test 1', (<<~EOF)) | |
version: 2 | |
updates: | |
# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8). | |
- package-ecosystem: "npm" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
# Apply this opt-in config by default to every project with a package.json. | |
apply-to-glob: "**/package.json" | |
exclude-regex: "node_modules" | |
EOF | |
if cfg['updates'].any? { |target| target['directory'].include?('node_modules') } | |
raise "Doesn't exclude node_modules" | |
end | |
raise "Doesn't seem to be finding all the files" if cfg['updates'].length < 20 | |
cfg = generate_config('test 2', (<<~EOF)) | |
version: 2 | |
updates: | |
# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8). | |
- package-ecosystem: "bundler" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
apply-to-glob: "**/Gemfile" | |
exclude-regex: "localgems" | |
ignore: | |
# TODO: DRP to provide a reason for this. It was blocked for unspecified reasons in https://github.com/StileEducation/dev-environment/pull/6335. | |
# UPDATE CONDITION: Requires investigation. | |
- dependency-name: "mongo" | |
blocked-reason: "bad-upstream-version" | |
blocked-reason-detail: "We just don't like it, Sam I am" | |
EOF | |
if cfg['updates'].any? do |target| | |
target['ignore'].first['dependency-name'] != 'mongo' | |
end | |
raise "Doesn't keep ignore" | |
end | |
cfg = generate_config('test 3', (<<~EOF)) | |
version: 2 | |
updates: | |
# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8). | |
- package-ecosystem: "npm" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
# Apply this opt-in config by default to every project with a package.json. | |
apply-to-glob: "**/package.json" | |
exclude-regex: "node_modules" | |
- package-ecosystem: "npm" | |
# The web-client itself has a lot of special-case restrictions. | |
apply-to-glob: "web-client/package.json" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
ignore: | |
# UPDATE CONDITION: IE11 EOL. | |
- dependency-name: "uuid" | |
blocked-reason: "bad-upstream-version" | |
blocked-reason-detail: "Breaks IE11 for incomprehensible reasons" | |
- package-ecosystem: "docker" | |
apply-to-glob: "web-client/Dockerfile" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
EOF | |
unless cfg['updates'].any? do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'npm' | |
end | |
raise 'Docker package stomped JavaScript package' | |
end | |
unless cfg['updates'].any? do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'docker' | |
end | |
raise 'Web-client package stomped Docker package' | |
end | |
unless cfg['updates'].select do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'npm' | |
end.length == 1 | |
raise 'Found duplicate entries for the same directory and package manager' | |
end | |
unless cfg['updates'].select do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'npm' | |
end.first[ | |
'ignore' | |
].first[ | |
'dependency-name' | |
] == 'uuid' | |
raise "Preserved directory isn't the later one" | |
end | |
# While developing this script, you may find it useful to uncomment the following line to only run the tests. | |
# puts "PASSED TESTS"; exit 0 | |
# For bonus points, feel free to run in 'watch mode' using the following wizardly incantation :) | |
# fswatch generate_dependabot_config.rb | xargs -n1 -I{} generate_dependabot_config.rb | |
### ACTUAL SCRIPT INVOCATION ### | |
generated_config = | |
generate_config( | |
CONFIG_GENERATOR_CONFIG_FILENAME, | |
File.read(CONFIG_GENERATOR_CONFIG_FILENAME), | |
) | |
File.open(DEPENDABOT_ACTUAL_CONFIG_FILENAME, 'w') do |f| | |
f.puts( | |
'# DO NOT MODIFY THIS FILE BY HAND, IT IS GENERATED BY bin/generate_dependabot_config.rb', | |
) | |
f.puts( | |
YAML.dump( | |
# Why, you may wonder, do I convert this generated_config to json and then parse it... before outputting it as yaml? | |
# No, I don't serialize things in multiple formats and back just for fun. The Ruby YAML output will 'cleverly' use advanced YAML syntax, 'aliases', | |
# to represent objects with the same identity in memory. Neat as this is, the Dependabot validator says it doesn't support that syntax. | |
# This re-serialisation avoids that issue by forcing Ruby to make different objects in memory for every entry. | |
JSON.parse(generated_config.to_json), | |
), | |
) | |
end | |
puts "Wrote out new dependabot config to #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}" | |
puts 'Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/' | |
puts "Checking that the config we generated passes Dependabot's validator API" | |
response = | |
HTTParty.post( | |
'https://api.dependabot.com/config_files/validate', | |
body: { | |
'config-file-body': generated_config.to_json, | |
}, | |
) | |
pp response.parsed_response | |
unless response.parsed_response['errors'].empty? | |
raise 'Dependabot says this config is invalid' | |
end | |
# If you use prettier, you may want to uncomment the following lines :) | |
# begin | |
# `prettier --write #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}` | |
# rescue => e | |
# puts "Unable to run prettier: #{e.inspect}" | |
# end |
@alexf101 I've just stumbled across your script, thanks so much for the effort! I'm having trouble getting the unit tests to pass too.
Environment
- macOS 11.6.4
- ruby:
ruby --version
ruby 2.6.3p62 (2019-04-16 revision 67580) [universal.x86_64-darwin20]
CONFIG_GENERATOR_CONFIG_FILENAME
/Users/xxx/xxx/xxx/dependabot_config_generator_config.yml
(I had to use the fully qualified system path)
DEPENDABOT_ACTUAL_CONFIG_FILENAME
/Users/xxx/xxx/xxx/github/dependabot.yml
(I had to use the fully qualified system path)
Unit test errors
./generate_dependabot_config.rb
Traceback (most recent call last):
./generate_dependabot_config.rb:268:in `<main>': Docker package stomped JavaScript package (RuntimeError)
Error if I comment out the unit tests
./generate_dependabot_config.rb
Wrote out new dependabot config to /Users/jasonbrewer/workshop/udx-api-integrators/.github/dependabot.yml
Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/
Checking that the config we generated passes Dependabot's validator API
"Not Found"
Traceback (most recent call last):
./generate_dependabot_config.rb:332:in `<main>': undefined method `empty?' for nil:NilClass (NoMethodError)
I'm not that au fait with Ruby but I suspect this might be a Ruby v2 vs Ruby v3 thing?
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hey. Should be as simple as configuring the following two variables at the top:
CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml'
DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml'
Your config file at CONFIG_GENERATOR_CONFIG_FILENAME should look basically like a Dependabot config, except that it will specify 'apply-to-glob' instead of a path to a file.
Example:
The script also has some unit tests built into it to make it easier to modify without breaking it. I'm surprised to hear that they're not passing - @thedmeyer would you mind posting the error message?