Translates a Dependabot config with glob support to a real Dependabot config

generate_dependabot_config.rb

#!/usr/bin/env ruby
require 'bundler/inline'

gemfile do
    source 'https://rubygems.org'
    gem 'httparty'
    gem 'fast_ignore'
end

require 'httparty'
require 'yaml'
require 'set'
require 'json'
require 'pp'

CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml'
DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml'
REPO_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
Dir.chdir REPO_ROOT

GLOBAL_EXCLUDE_REGEXES = [
    %r{\/node_modules\/},
]

# Validates the special config file that understands Stile's extensions.
def validate_generator_config(config_name, config)
    raise 'Missing updates?' unless config['updates'].length >= 1
    config['updates'].each do |target|
        raise 'Missing package-ecosystem' unless target['package-ecosystem']
        raise 'Missing schedule' unless target['schedule']
        raise 'Missing apply-to-glob' unless target['apply-to-glob']
        target
            .fetch('ignore', [])
            .each do |ignored_update|
                dependency_name = ignored_update['dependency-name']
                unless dependency_name
                    raise "Missing dependency name: #{target.pretty_inspect}"
                end
                unless ignored_update['blocked-reason']
                    raise "#{dependency_name}: Missing blocked-reason"
                end

                # This whitelist of valid values for blocked-reason intentionally makes it a bit difficult to add new ones willy-nilly. Please consider carefully whether whatever the new reason is, is actually a good enough reason to block updates.
                unless %w[
                           bad-upstream-version
                           deletion-planned
                           requires-other-project:drop-ie11
                           requires-other-project:delete-backbone
                       ].include?(ignored_update['blocked-reason'])
                    raise "#{dependency_name}: Invalid value for blocked-reason"
                end
                unless ignored_update['blocked-reason-detail']
                    raise "#{dependency_name}: Missing blocked-reason-detail"
                end
            end
    end
    config
rescue StandardError
    STDERR.puts "Validation failed for input config file at `#{config_name}`"
    raise
end

# Validates the generated vanilla Dependabot config files (to let us test that we didn't stuff up the logic)
def validate_generated_config(config_name, generated_config)
    raise 'Missing version' unless generated_config['version']
    raise 'Missing updates' unless generated_config['updates']
    generated_config['updates'].each do |target|
        raise 'Missing directory' unless target['directory']
        valid_keys = %w[
            package-ecosystem
            directory
            schedule
            allow
            assignees
            commit-message
            ignore
            insecure-external-code-execution
            labels
            milestone
            open-pull-requests-limit
            pull-request-branch-name
            rebase-strategy
            registries
            reviewers
            target-branch
            vendor
            versioning-strategy
        ]
        invalid_keys = target.keys - valid_keys
        if invalid_keys.length > 0
            raise "Contains invalid keys: #{invalid_keys}"
        end
        target
            .fetch('ignore', [])
            .each do |ignored_update|
                dependency_name = ignored_update['dependency-name']
                raise 'Missing dependency name' unless dependency_name
                invalid_keys =
                    ignored_update.keys - %w[
                        dependency-name
                        version_requirement
                    ]
                if invalid_keys.length > 0
                    raise "Contains invalid keys: #{invalid_keys}"
                end
            end
    end
    generated_config
rescue StandardError
    STDERR.puts "Validation failed for config file `#{config_name}`"
    raise
end

# Core logic of this script - translates our extensions into a vanilla Dependabot format through judicious use of glob and regex.
def generate_config(config_name, config_generator_config)
    config = YAML.load(config_generator_config)
    validate_generator_config(config_name, config)
    new_config = YAML.load(config_generator_config)
    new_config['updates'] = []
    config['updates'].each do |target|
        apply_to_glob = target['apply-to-glob']
        exclude_regex = target['exclude-regex']

        # Find all matching glob files, while applying gitignore rules
        matched_files =
            FastIgnore.new(
                relative: true,
                argv_rules: [apply_to_glob],
                root: REPO_ROOT,
            ).to_a.select do |path|
                !GLOBAL_EXCLUDE_REGEXES.any? do |global_exclude_regex|
                    global_exclude_regex.match?(path)
                end
            end.sort

        if exclude_regex
            exclude_regex_parsed = Regexp.new(exclude_regex)
            matched_files =
                matched_files.reject do |path|
                    exclude_regex_parsed.match?(path)
                end
        end

        if target['ignore']
            target['ignore'].each do |ignored_update|
                ignored_update.reject! do |k|
                    %w[blocked-reason blocked-reason-detail].include?(k)
                end
            end
        end
        matched_files.each do |filepath|
            new_config['updates'] <<
                {
                    # Directory is relative to the repository's root, with '/' indicating the repo's root. However, it also seems to find '.' and relative paths without '/' acceptable, so... let's just do that then.
                    # See https://dependabot.com/docs/config-file/validator/ for confirmation.
                    'directory' => File.dirname(filepath),
                }.merge(
                    target.reject do |k|
                        %w[apply-to-glob exclude-regex].include?(k)
                    end,
                )
        end

        # Remove duplicates - later entries should override earlier ones.
        found_so_far = Set.new
        new_config['updates'] =
            new_config['updates'].reverse.reject do |target|
                unique_key = [target['directory'], target['package-ecosystem']]
                should_skip = found_so_far.include?(unique_key)
                found_so_far << unique_key
                should_skip
            end.sort_by do |target|
                # Stable output important to minimise human-readable diff.
                #
                # There is no "primary key" for entries in the Dependabot
                # config schema (https://dependabot.com/docs/config-file/)
                # but it seems unlikely that we'll have multiple entries
                # that share the same directory and package manager -- e.g.
                # updating exactly the same stuff but on a different schedule.
                [target['directory'], target['package-ecosystem']]
            end
    end
    validate_generated_config(config_name, new_config)
    new_config
end

### UNIT TESTS ###
# May as well run them every time the script runs for a wee little project like this.
cfg = generate_config('test 1', (<<~EOF))
    version: 2
    updates:
        # The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
        - package-ecosystem: "npm"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          # Apply this opt-in config by default to every project with a package.json.
          apply-to-glob: "**/package.json"
          exclude-regex: "node_modules"
EOF

if cfg['updates'].any? { |target| target['directory'].include?('node_modules') }
    raise "Doesn't exclude node_modules"
end
raise "Doesn't seem to be finding all the files" if cfg['updates'].length < 20
cfg = generate_config('test 2', (<<~EOF))
    version: 2
    updates:
        # The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
        - package-ecosystem: "bundler"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          apply-to-glob: "**/Gemfile"
          exclude-regex: "localgems"
          ignore:
            # TODO: DRP to provide a reason for this. It was blocked for unspecified reasons in https://github.com/StileEducation/dev-environment/pull/6335.
            # UPDATE CONDITION: Requires investigation.
            - dependency-name: "mongo"
              blocked-reason: "bad-upstream-version"
              blocked-reason-detail: "We just don't like it, Sam I am"
EOF

if cfg['updates'].any? do |target|
       target['ignore'].first['dependency-name'] != 'mongo'
   end
    raise "Doesn't keep ignore"
end

cfg = generate_config('test 3', (<<~EOF))
    version: 2
    updates:
        # The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
        - package-ecosystem: "npm"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          # Apply this opt-in config by default to every project with a package.json.
          apply-to-glob: "**/package.json"
          exclude-regex: "node_modules"
        - package-ecosystem: "npm"
          # The web-client itself has a lot of special-case restrictions.
          apply-to-glob: "web-client/package.json"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          ignore:
              # UPDATE CONDITION: IE11 EOL.
              - dependency-name: "uuid"
                blocked-reason: "bad-upstream-version"
                blocked-reason-detail: "Breaks IE11 for incomprehensible reasons"
        - package-ecosystem: "docker"
          apply-to-glob: "web-client/Dockerfile"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
EOF

unless cfg['updates'].any? do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'npm'
       end
    raise 'Docker package stomped JavaScript package'
end
unless cfg['updates'].any? do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'docker'
       end
    raise 'Web-client package stomped Docker package'
end
unless cfg['updates'].select do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'npm'
       end.length == 1
    raise 'Found duplicate entries for the same directory and package manager'
end
unless cfg['updates'].select do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'npm'
       end.first[
           'ignore'
       ].first[
           'dependency-name'
       ] == 'uuid'
    raise "Preserved directory isn't the later one"
end

# While developing this script, you may find it useful to uncomment the following line to only run the tests.
# puts "PASSED TESTS"; exit 0
# For bonus points, feel free to run in 'watch mode' using the following wizardly incantation :)
# fswatch generate_dependabot_config.rb | xargs -n1 -I{} generate_dependabot_config.rb

### ACTUAL SCRIPT INVOCATION ###

generated_config =
    generate_config(
        CONFIG_GENERATOR_CONFIG_FILENAME,
        File.read(CONFIG_GENERATOR_CONFIG_FILENAME),
    )

File.open(DEPENDABOT_ACTUAL_CONFIG_FILENAME, 'w') do |f|
    f.puts(
        '# DO NOT MODIFY THIS FILE BY HAND, IT IS GENERATED BY bin/generate_dependabot_config.rb',
    )
    f.puts(
        YAML.dump(
            # Why, you may wonder, do I convert this generated_config to json and then parse it... before outputting it as yaml?
            # No, I don't serialize things in multiple formats and back just for fun. The Ruby YAML output will 'cleverly' use advanced YAML syntax, 'aliases',
            # to represent objects with the same identity in memory. Neat as this is, the Dependabot validator says it doesn't support that syntax.
            # This re-serialisation avoids that issue by forcing Ruby to make different objects in memory for every entry.
            JSON.parse(generated_config.to_json),
        ),
    )
end
puts "Wrote out new dependabot config to #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}"
puts 'Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/'

puts "Checking that the config we generated passes Dependabot's validator API"
response =
    HTTParty.post(
        'https://api.dependabot.com/config_files/validate',
        body: {
            'config-file-body': generated_config.to_json,
        },
    )
pp response.parsed_response
unless response.parsed_response['errors'].empty?
    raise 'Dependabot says this config is invalid'
end

# If you use prettier, you may want to uncomment the following lines :)
# begin
#     `prettier --write #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}`
# rescue => e
#     puts "Unable to run prettier: #{e.inspect}"
# end

Could you provide some more insight on this script? @alexf101

I'm having trouble getting it to pass unit tests

Thanks!

Hey. Should be as simple as configuring the following two variables at the top:

CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml'
DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml'

Your config file at CONFIG_GENERATOR_CONFIG_FILENAME should look basically like a Dependabot config, except that it will specify 'apply-to-glob' instead of a path to a file.

Example:

version: 2
updates:
    - package-ecosystem: "pip"
      apply-to-glob: "**/requirements.txt"
      open-pull-requests-limit: 99
      schedule:
          interval: "daily"
          time: "02:00"
          timezone: Australia/Melbourne

The script also has some unit tests built into it to make it easier to modify without breaking it. I'm surprised to hear that they're not passing - @thedmeyer would you mind posting the error message?

@alexf101 I've just stumbled across your script, thanks so much for the effort! I'm having trouble getting the unit tests to pass too.

Environment

• macOS 11.6.4
• ruby:

ruby --version
ruby 2.6.3p62 (2019-04-16 revision 67580) [universal.x86_64-darwin20]

CONFIG_GENERATOR_CONFIG_FILENAME

/Users/xxx/xxx/xxx/dependabot_config_generator_config.yml

(I had to use the fully qualified system path)

DEPENDABOT_ACTUAL_CONFIG_FILENAME

/Users/xxx/xxx/xxx/github/dependabot.yml

(I had to use the fully qualified system path)

Unit test errors

./generate_dependabot_config.rb
Traceback (most recent call last):
./generate_dependabot_config.rb:268:in `<main>': Docker package stomped JavaScript package (RuntimeError)

Error if I comment out the unit tests

./generate_dependabot_config.rb
Wrote out new dependabot config to /Users/jasonbrewer/workshop/udx-api-integrators/.github/dependabot.yml
Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/
Checking that the config we generated passes Dependabot's validator API
"Not Found"
Traceback (most recent call last):
./generate_dependabot_config.rb:332:in `<main>': undefined method `empty?' for nil:NilClass (NoMethodError)

I'm not that au fait with Ruby but I suspect this might be a Ruby v2 vs Ruby v3 thing?