Using dependabot's file_fetchers offline

ABOUT.md

Using dependabot's file_fetchers offline

The context for this example is wanting to parse an already existing local clone of a repository, without additional network traffic and without providing configuration to access a potentially private repository, if we already have a clone of it and want to use the functionality that dependabot provides of parsing for the files its ecosystems expect.

This provides an example of how to monkey patch the gem dependabot-common@0.212.0 to enable the result of the below example that wraps fetcher.files.map(&:name), provided in the file_fetchers README;

puts "Fetched #{fetcher.files.map(&:name)}, at commit SHA-1 '#{fetcher.commit}'"

A quick bundle install should set you up with the 0.212.x version of dependabot-common and dependabot-python, the last version to support Ruby version 2.7.0. As the intent of this is to demonstrate running the fetcher.files.map(&:name) offline, a local clone of the repository you want to run it on should exist already. I used my Skenvy/Collatz project, as being implemented in parallel in multiple languages made it a good target for scanning for the files that dependabot looks for, across multiple languages. This example demonstrates the dependabot-python@0.212.0 file_fetcher.

Running bundle exec ruby offline_dependabot_example.rb should yield a result along the lines of;

Fetched ["pyproject.toml", "requirements-venv.txt", "requirements.txt", "setup.py", "setup.cfg"], at commit SHA-1 '4c22c0675b5ddf694ece30e40d52f2b93ff4440d'

Be aware of dependabot-core's LICENSE, which is The Prosperity Public License 2.0.0, which limits commercial use.

Gemfile

# frozen_string_literal: true
source "https://rubygems.org"
gem "dependabot-omnibus", "~> 0.212.0"

Gemfile.lock

GEM
  remote: https://rubygems.org/
  specs:
    activesupport (7.0.4)
      concurrent-ruby (~> 1.0, >= 1.0.2)
      i18n (>= 1.6, < 2)
      minitest (>= 5.1)
      tzinfo (~> 2.0)
    addressable (2.8.1)
      public_suffix (>= 2.0.2, < 6.0)
    ast (2.4.2)
    aws-eventstream (1.2.0)
    aws-partitions (1.654.0)
    aws-sdk-codecommit (1.52.0)
      aws-sdk-core (~> 3, >= 3.165.0)
      aws-sigv4 (~> 1.1)
    aws-sdk-core (3.166.0)
      aws-eventstream (~> 1, >= 1.0.2)
      aws-partitions (~> 1, >= 1.651.0)
      aws-sigv4 (~> 1.5)
      jmespath (~> 1, >= 1.6.1)
    aws-sdk-ecr (1.57.0)
      aws-sdk-core (~> 3, >= 3.165.0)
      aws-sigv4 (~> 1.1)
    aws-sigv4 (1.5.2)
      aws-eventstream (~> 1, >= 1.0.2)
    citrus (3.0.2)
    commonmarker (0.23.6)
    concurrent-ruby (1.1.10)
    dependabot-bundler (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-cargo (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-common (0.212.0)
      activesupport (>= 6.0.0)
      aws-sdk-codecommit (~> 1.28)
      aws-sdk-ecr (~> 1.5)
      bundler (>= 1.16, < 3.0.0)
      commonmarker (>= 0.20.1, < 0.24.0)
      docker_registry2 (~> 1.11, >= 1.11.0)
      excon (~> 0.75)
      faraday (= 2.5.2)
      gitlab (= 4.19.0)
      nokogiri (~> 1.8)
      octokit (>= 4.6, < 6.0)
      parser (>= 2.5, < 4.0)
      toml-rb (>= 1.1.2, < 3.0)
    dependabot-composer (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-docker (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-elm (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-git_submodules (0.212.0)
      dependabot-common (= 0.212.0)
      parseconfig (~> 1.0, < 1.1.0)
    dependabot-github_actions (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-go_modules (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-gradle (0.212.0)
      dependabot-common (= 0.212.0)
      dependabot-maven (= 0.212.0)
    dependabot-hex (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-maven (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-npm_and_yarn (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-nuget (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-omnibus (0.212.0)
      dependabot-bundler (= 0.212.0)
      dependabot-cargo (= 0.212.0)
      dependabot-common (= 0.212.0)
      dependabot-composer (= 0.212.0)
      dependabot-docker (= 0.212.0)
      dependabot-elm (= 0.212.0)
      dependabot-git_submodules (= 0.212.0)
      dependabot-github_actions (= 0.212.0)
      dependabot-go_modules (= 0.212.0)
      dependabot-gradle (= 0.212.0)
      dependabot-hex (= 0.212.0)
      dependabot-maven (= 0.212.0)
      dependabot-npm_and_yarn (= 0.212.0)
      dependabot-nuget (= 0.212.0)
      dependabot-pub (= 0.212.0)
      dependabot-python (= 0.212.0)
      dependabot-terraform (= 0.212.0)
    dependabot-pub (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-python (0.212.0)
      dependabot-common (= 0.212.0)
    dependabot-terraform (0.212.0)
      dependabot-common (= 0.212.0)
    docker_registry2 (1.12.0)
      rest-client (>= 1.8.0)
    domain_name (0.5.20190701)
      unf (>= 0.0.5, < 1.0.0)
    excon (0.93.1)
    faraday (2.5.2)
      faraday-net_http (>= 2.0, < 3.1)
      ruby2_keywords (>= 0.0.4)
    faraday-net_http (3.0.1)
    gitlab (4.19.0)
      httparty (~> 0.20)
      terminal-table (>= 1.5.1)
    http-accept (1.7.0)
    http-cookie (1.0.5)
      domain_name (~> 0.5)
    httparty (0.20.0)
      mime-types (~> 3.0)
      multi_xml (>= 0.5.2)
    i18n (1.12.0)
      concurrent-ruby (~> 1.0)
    jmespath (1.6.1)
    mime-types (3.4.1)
      mime-types-data (~> 3.2015)
    mime-types-data (3.2022.0105)
    minitest (5.16.3)
    multi_xml (0.6.0)
    netrc (0.11.0)
    nokogiri (1.13.9-x86_64-linux)
      racc (~> 1.4)
    octokit (5.6.1)
      faraday (>= 1, < 3)
      sawyer (~> 0.9)
    parseconfig (1.0.8)
    parser (3.1.2.1)
      ast (~> 2.4.1)
    public_suffix (5.0.0)
    racc (1.6.0)
    rest-client (2.1.0)
      http-accept (>= 1.7.0, < 2.0)
      http-cookie (>= 1.0.2, < 2.0)
      mime-types (>= 1.16, < 4.0)
      netrc (~> 0.8)
    ruby2_keywords (0.0.5)
    sawyer (0.9.2)
      addressable (>= 2.3.5)
      faraday (>= 0.17.3, < 3)
    terminal-table (3.0.2)
      unicode-display_width (>= 1.1.1, < 3)
    toml-rb (2.2.0)
      citrus (~> 3.0, > 3.0)
    tzinfo (2.0.5)
      concurrent-ruby (~> 1.0)
    unf (0.1.4)
      unf_ext
    unf_ext (0.0.8.2)
    unicode-display_width (2.3.0)

PLATFORMS
  x86_64-linux

DEPENDENCIES
  dependabot-omnibus (~> 0.212.0)

BUNDLED WITH
   2.3.24

offline_dependabot_example.rb

# Uses dependabot 0.212.0, because 0.213.0 doesn't support ruby 2.7,
# but the Dependabot::FileFetchers::Base needs a monkey patch.

require 'dependabot/source'
require 'dependabot/file_fetchers'
require 'dependabot/python'

target_provider = 'github'
target_repo = 'Skenvy/Collatz'
target_pre_cloned_path = '../Collatz'
target_directory = '/python'
target_package_manager = 'pip'

source = Dependabot::Source.new(provider: target_provider, repo: target_repo, directory: target_directory)
fetcher_class = Dependabot::FileFetchers::for_package_manager(target_package_manager)

module Dependabot
  module FileFetchers
    class Base
      # attr_accessor :clone_repo_contents
      def cloned_commit
        return if repo_contents_path.nil? || !File.directory?(File.join(repo_contents_path, ".git"))
        Dir.chdir(repo_contents_path) do
          return SharedHelpers.run_shell_command("git rev-parse HEAD")&.strip
        end
      end
      def commit
        return cloned_commit if cloned_commit
        return source.commit if source.commit
        branch = target_branch || default_branch_for_repo
        @commit ||= client_for_provider.fetch_commit(repo, branch)
      rescue *CLIENT_NOT_FOUND_ERRORS
        raise Dependabot::BranchNotFound, branch
      rescue Octokit::Conflict => e
        raise unless e.message.include?("Repository is empty")
      end
    end
  end
end

fetcher = fetcher_class.new(source: source, credentials: [], repo_contents_path: target_pre_cloned_path)
# fetcher.clone_repo_contents = target_pre_cloned_path

puts "Fetched #{fetcher.files.map(&:name)}, at commit SHA-1 '#{fetcher.commit}'"