-
-
Save alexf101/b65cbfe7c5a61df7d925589a71d200cf to your computer and use it in GitHub Desktop.
#!/usr/bin/env ruby | |
require 'bundler/inline' | |
gemfile do | |
source 'https://rubygems.org' | |
gem 'httparty' | |
gem 'fast_ignore' | |
end | |
require 'httparty' | |
require 'yaml' | |
require 'set' | |
require 'json' | |
require 'pp' | |
CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml' | |
DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml' | |
REPO_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) | |
Dir.chdir REPO_ROOT | |
GLOBAL_EXCLUDE_REGEXES = [ | |
%r{\/node_modules\/}, | |
] | |
# Validates the special config file that understands Stile's extensions. | |
def validate_generator_config(config_name, config) | |
raise 'Missing updates?' unless config['updates'].length >= 1 | |
config['updates'].each do |target| | |
raise 'Missing package-ecosystem' unless target['package-ecosystem'] | |
raise 'Missing schedule' unless target['schedule'] | |
raise 'Missing apply-to-glob' unless target['apply-to-glob'] | |
target | |
.fetch('ignore', []) | |
.each do |ignored_update| | |
dependency_name = ignored_update['dependency-name'] | |
unless dependency_name | |
raise "Missing dependency name: #{target.pretty_inspect}" | |
end | |
unless ignored_update['blocked-reason'] | |
raise "#{dependency_name}: Missing blocked-reason" | |
end | |
# This whitelist of valid values for blocked-reason intentionally makes it a bit difficult to add new ones willy-nilly. Please consider carefully whether whatever the new reason is, is actually a good enough reason to block updates. | |
unless %w[ | |
bad-upstream-version | |
deletion-planned | |
requires-other-project:drop-ie11 | |
requires-other-project:delete-backbone | |
].include?(ignored_update['blocked-reason']) | |
raise "#{dependency_name}: Invalid value for blocked-reason" | |
end | |
unless ignored_update['blocked-reason-detail'] | |
raise "#{dependency_name}: Missing blocked-reason-detail" | |
end | |
end | |
end | |
config | |
rescue StandardError | |
STDERR.puts "Validation failed for input config file at `#{config_name}`" | |
raise | |
end | |
# Validates the generated vanilla Dependabot config files (to let us test that we didn't stuff up the logic) | |
def validate_generated_config(config_name, generated_config) | |
raise 'Missing version' unless generated_config['version'] | |
raise 'Missing updates' unless generated_config['updates'] | |
generated_config['updates'].each do |target| | |
raise 'Missing directory' unless target['directory'] | |
valid_keys = %w[ | |
package-ecosystem | |
directory | |
schedule | |
allow | |
assignees | |
commit-message | |
ignore | |
insecure-external-code-execution | |
labels | |
milestone | |
open-pull-requests-limit | |
pull-request-branch-name | |
rebase-strategy | |
registries | |
reviewers | |
target-branch | |
vendor | |
versioning-strategy | |
] | |
invalid_keys = target.keys - valid_keys | |
if invalid_keys.length > 0 | |
raise "Contains invalid keys: #{invalid_keys}" | |
end | |
target | |
.fetch('ignore', []) | |
.each do |ignored_update| | |
dependency_name = ignored_update['dependency-name'] | |
raise 'Missing dependency name' unless dependency_name | |
invalid_keys = | |
ignored_update.keys - %w[ | |
dependency-name | |
version_requirement | |
] | |
if invalid_keys.length > 0 | |
raise "Contains invalid keys: #{invalid_keys}" | |
end | |
end | |
end | |
generated_config | |
rescue StandardError | |
STDERR.puts "Validation failed for config file `#{config_name}`" | |
raise | |
end | |
# Core logic of this script - translates our extensions into a vanilla Dependabot format through judicious use of glob and regex. | |
def generate_config(config_name, config_generator_config) | |
config = YAML.load(config_generator_config) | |
validate_generator_config(config_name, config) | |
new_config = YAML.load(config_generator_config) | |
new_config['updates'] = [] | |
config['updates'].each do |target| | |
apply_to_glob = target['apply-to-glob'] | |
exclude_regex = target['exclude-regex'] | |
# Find all matching glob files, while applying gitignore rules | |
matched_files = | |
FastIgnore.new( | |
relative: true, | |
argv_rules: [apply_to_glob], | |
root: REPO_ROOT, | |
).to_a.select do |path| | |
!GLOBAL_EXCLUDE_REGEXES.any? do |global_exclude_regex| | |
global_exclude_regex.match?(path) | |
end | |
end.sort | |
if exclude_regex | |
exclude_regex_parsed = Regexp.new(exclude_regex) | |
matched_files = | |
matched_files.reject do |path| | |
exclude_regex_parsed.match?(path) | |
end | |
end | |
if target['ignore'] | |
target['ignore'].each do |ignored_update| | |
ignored_update.reject! do |k| | |
%w[blocked-reason blocked-reason-detail].include?(k) | |
end | |
end | |
end | |
matched_files.each do |filepath| | |
new_config['updates'] << | |
{ | |
# Directory is relative to the repository's root, with '/' indicating the repo's root. However, it also seems to find '.' and relative paths without '/' acceptable, so... let's just do that then. | |
# See https://dependabot.com/docs/config-file/validator/ for confirmation. | |
'directory' => File.dirname(filepath), | |
}.merge( | |
target.reject do |k| | |
%w[apply-to-glob exclude-regex].include?(k) | |
end, | |
) | |
end | |
# Remove duplicates - later entries should override earlier ones. | |
found_so_far = Set.new | |
new_config['updates'] = | |
new_config['updates'].reverse.reject do |target| | |
unique_key = [target['directory'], target['package-ecosystem']] | |
should_skip = found_so_far.include?(unique_key) | |
found_so_far << unique_key | |
should_skip | |
end.sort_by do |target| | |
# Stable output important to minimise human-readable diff. | |
# | |
# There is no "primary key" for entries in the Dependabot | |
# config schema (https://dependabot.com/docs/config-file/) | |
# but it seems unlikely that we'll have multiple entries | |
# that share the same directory and package manager -- e.g. | |
# updating exactly the same stuff but on a different schedule. | |
[target['directory'], target['package-ecosystem']] | |
end | |
end | |
validate_generated_config(config_name, new_config) | |
new_config | |
end | |
### UNIT TESTS ### | |
# May as well run them every time the script runs for a wee little project like this. | |
cfg = generate_config('test 1', (<<~EOF)) | |
version: 2 | |
updates: | |
# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8). | |
- package-ecosystem: "npm" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
# Apply this opt-in config by default to every project with a package.json. | |
apply-to-glob: "**/package.json" | |
exclude-regex: "node_modules" | |
EOF | |
if cfg['updates'].any? { |target| target['directory'].include?('node_modules') } | |
raise "Doesn't exclude node_modules" | |
end | |
raise "Doesn't seem to be finding all the files" if cfg['updates'].length < 20 | |
cfg = generate_config('test 2', (<<~EOF)) | |
version: 2 | |
updates: | |
# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8). | |
- package-ecosystem: "bundler" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
apply-to-glob: "**/Gemfile" | |
exclude-regex: "localgems" | |
ignore: | |
# TODO: DRP to provide a reason for this. It was blocked for unspecified reasons in https://github.com/StileEducation/dev-environment/pull/6335. | |
# UPDATE CONDITION: Requires investigation. | |
- dependency-name: "mongo" | |
blocked-reason: "bad-upstream-version" | |
blocked-reason-detail: "We just don't like it, Sam I am" | |
EOF | |
if cfg['updates'].any? do |target| | |
target['ignore'].first['dependency-name'] != 'mongo' | |
end | |
raise "Doesn't keep ignore" | |
end | |
cfg = generate_config('test 3', (<<~EOF)) | |
version: 2 | |
updates: | |
# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8). | |
- package-ecosystem: "npm" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
# Apply this opt-in config by default to every project with a package.json. | |
apply-to-glob: "**/package.json" | |
exclude-regex: "node_modules" | |
- package-ecosystem: "npm" | |
# The web-client itself has a lot of special-case restrictions. | |
apply-to-glob: "web-client/package.json" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
ignore: | |
# UPDATE CONDITION: IE11 EOL. | |
- dependency-name: "uuid" | |
blocked-reason: "bad-upstream-version" | |
blocked-reason-detail: "Breaks IE11 for incomprehensible reasons" | |
- package-ecosystem: "docker" | |
apply-to-glob: "web-client/Dockerfile" | |
schedule: | |
interval: "daily" | |
time: "02:00" | |
timezone: "Australia/Melbourne" | |
EOF | |
unless cfg['updates'].any? do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'npm' | |
end | |
raise 'Docker package stomped JavaScript package' | |
end | |
unless cfg['updates'].any? do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'docker' | |
end | |
raise 'Web-client package stomped Docker package' | |
end | |
unless cfg['updates'].select do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'npm' | |
end.length == 1 | |
raise 'Found duplicate entries for the same directory and package manager' | |
end | |
unless cfg['updates'].select do |target| | |
target['directory'] == 'web-client' && | |
target['package-ecosystem'] == 'npm' | |
end.first[ | |
'ignore' | |
].first[ | |
'dependency-name' | |
] == 'uuid' | |
raise "Preserved directory isn't the later one" | |
end | |
# While developing this script, you may find it useful to uncomment the following line to only run the tests. | |
# puts "PASSED TESTS"; exit 0 | |
# For bonus points, feel free to run in 'watch mode' using the following wizardly incantation :) | |
# fswatch generate_dependabot_config.rb | xargs -n1 -I{} generate_dependabot_config.rb | |
### ACTUAL SCRIPT INVOCATION ### | |
generated_config = | |
generate_config( | |
CONFIG_GENERATOR_CONFIG_FILENAME, | |
File.read(CONFIG_GENERATOR_CONFIG_FILENAME), | |
) | |
File.open(DEPENDABOT_ACTUAL_CONFIG_FILENAME, 'w') do |f| | |
f.puts( | |
'# DO NOT MODIFY THIS FILE BY HAND, IT IS GENERATED BY bin/generate_dependabot_config.rb', | |
) | |
f.puts( | |
YAML.dump( | |
# Why, you may wonder, do I convert this generated_config to json and then parse it... before outputting it as yaml? | |
# No, I don't serialize things in multiple formats and back just for fun. The Ruby YAML output will 'cleverly' use advanced YAML syntax, 'aliases', | |
# to represent objects with the same identity in memory. Neat as this is, the Dependabot validator says it doesn't support that syntax. | |
# This re-serialisation avoids that issue by forcing Ruby to make different objects in memory for every entry. | |
JSON.parse(generated_config.to_json), | |
), | |
) | |
end | |
puts "Wrote out new dependabot config to #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}" | |
puts 'Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/' | |
puts "Checking that the config we generated passes Dependabot's validator API" | |
response = | |
HTTParty.post( | |
'https://api.dependabot.com/config_files/validate', | |
body: { | |
'config-file-body': generated_config.to_json, | |
}, | |
) | |
pp response.parsed_response | |
unless response.parsed_response['errors'].empty? | |
raise 'Dependabot says this config is invalid' | |
end | |
# If you use prettier, you may want to uncomment the following lines :) | |
# begin | |
# `prettier --write #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}` | |
# rescue => e | |
# puts "Unable to run prettier: #{e.inspect}" | |
# end |
Hey. Should be as simple as configuring the following two variables at the top:
CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml'
DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml'
Your config file at CONFIG_GENERATOR_CONFIG_FILENAME should look basically like a Dependabot config, except that it will specify 'apply-to-glob' instead of a path to a file.
Example:
version: 2
updates:
- package-ecosystem: "pip"
apply-to-glob: "**/requirements.txt"
open-pull-requests-limit: 99
schedule:
interval: "daily"
time: "02:00"
timezone: Australia/Melbourne
The script also has some unit tests built into it to make it easier to modify without breaking it. I'm surprised to hear that they're not passing - @thedmeyer would you mind posting the error message?
@alexf101 I've just stumbled across your script, thanks so much for the effort! I'm having trouble getting the unit tests to pass too.
Environment
- macOS 11.6.4
- ruby:
ruby --version
ruby 2.6.3p62 (2019-04-16 revision 67580) [universal.x86_64-darwin20]
CONFIG_GENERATOR_CONFIG_FILENAME
/Users/xxx/xxx/xxx/dependabot_config_generator_config.yml
(I had to use the fully qualified system path)
DEPENDABOT_ACTUAL_CONFIG_FILENAME
/Users/xxx/xxx/xxx/github/dependabot.yml
(I had to use the fully qualified system path)
Unit test errors
./generate_dependabot_config.rb
Traceback (most recent call last):
./generate_dependabot_config.rb:268:in `<main>': Docker package stomped JavaScript package (RuntimeError)
Error if I comment out the unit tests
./generate_dependabot_config.rb
Wrote out new dependabot config to /Users/jasonbrewer/workshop/udx-api-integrators/.github/dependabot.yml
Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/
Checking that the config we generated passes Dependabot's validator API
"Not Found"
Traceback (most recent call last):
./generate_dependabot_config.rb:332:in `<main>': undefined method `empty?' for nil:NilClass (NoMethodError)
I'm not that au fait with Ruby but I suspect this might be a Ruby v2 vs Ruby v3 thing?
Could you provide some more insight on this script? @alexf101
I'm having trouble getting it to pass unit tests
Thanks!