List non-CPAN prereqs

list-noncpan-prereqs.pl

#!/usr/bin/env perl
use 5.014;
use warnings;

use Carp::Always;
use ElasticSearch;
use File::Spec::Functions qw(catfile tmpdir);
use LWP::UserAgent;
use PerlIO::gzip;

use constant VERBOSE => 1;

sub d {
    my $time = sprintf '[%02d:%02d:%02d] ', (localtime)[2,1,0];
    print STDERR $time, @_, "\n";
}

my $es = ElasticSearch->new(
    servers    => 'api.metacpan.org',
    no_refresh => 1,
    deflate    => 1,
);

# Get the list of all released modules. Neither 02packages, nor metacpan
# contain a complete listing, so combine the results of both.

my %modules;
{
    my $file = '02packages.details.txt.gz';
    my $url = "http://search.cpan.org/CPAN/modules/$file";
    $file = catfile(tmpdir, $file);
    if (not -r $file or 1 < -M _) {
        my $ua = LWP::UserAgent->new;
        $ua->default_header(accept_encoding => undef);
        my $res = $ua->mirror($url, $file);
        die "Failed to mirror $file; " if $res->is_error;
    }

    open my $fh, '<:gzip', $file or die "$file: $!";
    # Skip the header.
    do 1 until <$fh> eq "\n";

    while (my $line = <$fh>) {
        my ($module, $version, $dist) = split /\s+/, $line;
        $modules{$module} = undef;
    }
}
d 'finished packages file' if VERBOSE;

my $req = $es->scrolled_search(
    index  => 'file',
    query  => { match_all => {} },
    filter => {
        and => [
            { term => { status     => 'cpan' } },
            { term => { maturity   => 'released' } },
            { term => { authorized => \1 } },
            # Taken from MetaCPAN::Web::Model::API::Release::modules
            {
                or => [
                    {
                        and => [
                            { exists => { field => 'file.module.name' } },
                            { term   => { 'file.module.indexed' => \1 } }
                        ]
                    },
                    {
                        and => [
                            { exists => { field => 'file.pod.analyzed' } },
                            { term   => { 'file.indexed' => \1 } }
                        ]
                    }
                ]
            }
        ],
    },
    fields      => '_source.module',
    search_type => 'scan',
    scroll      => '5m',
    size        => 1_000,
);

while (my $res = $req->next) {
    my $modules = $res->{fields}{'_source.module'} or next;
    for my $module (@$modules) {
        $modules{ $module->{name} } = undef;
    }
}
d 'finished metacpan modules list' if VERBOSE;

# Fetch the list of prereqs for all the latest distributions.
$req = $es->scrolled_search(
    index  => 'release',
    query  => { match_all => {} },
    filter => {
        and => [
            { term => { 'release.status'     => 'latest' } },
            { term => { 'release.authorized' => \1 } },
        ],
    },
    fields      => 'dependency',
    search_type => 'scan',
    scroll      => '5m',
    size        => 1_000,
);

my %prereqs;
while (my $res = $req->next) {
    my $deps = $res->{fields}{dependency} or next;
    for my $dep (@$deps) {
        next unless 'requires' eq $dep->{relationship} || '';
        $prereqs{ $dep->{module} } = undef;
    }
}
d 'finished metacpan prereqs list' if VERBOSE;

say for grep { not exists $modules{$_} } sort keys %prereqs;

list.txt

AI::MicroStructure::util
Acme::CPANAuthors::Italian
Acme::CPANPLUS::Module::With::Core::PreReq
Algorithm::LCS
Alien::Sedna
Angerwhale
Apache2
Apache2::Test
Apache::Const
Apache::RequestIO
Apache::RequestRec
Apache::RequestUtil
App::Gitc::Config
App::Gitc::ListFormat::Base
App::Horris
AptPkg::Cache
AptPkg::Config
BINGOS::BadModule
CGI::Wiki::Search::DB
CPANPLUS::Dist::RPM
CWB
CWB::CL
Catalyst::Example::InstantCRUD
Chatbot::TalkerBot
Class::DBI::Join
ClearSilver
Config::PlConfig
Crypt::C_LockTite
Crypt::CapnMidNite
DBIx::Class::Timestamp
DBIx::Migration::Directories::Build
DJabberd::Plugin::VCard
DMS
Data::Average
Data::JavaScript::Compactor
Data::NDS::Multiele
Data::NDS::Multifile
Data::NDS::Multisource
Data::Sah::Util
DataTime
Date::Calc::Util
Devel::Coverage
Devel::ebug::Backend
Dist::Zilla::PluginBundle::GopherRepellent
Dist::Zilla::PluginBundle::KENTNL::Lite
Dist::Zilla::PluginBundle::PadrePlugin
Dpkg
Dpkg::Changelog::Parse
Dpkg::Source::Package
ESL
Egg::Plugin::Cache
FVWM::Module
Fault::Errorhandler
File::Slurp::Remote
File::Slurp::Unicode
File::Tmp
File::find::Rule
Finance::Quote::Convert::WebserviceX
FirePHP::Dispatcher
Foundation
GLib
GPS::PRN
GSG::Eventum
Games::WoW::PVP
General::Parse
Geography::NationalGrid
Geography::NationalGrid::GB
Goose
Graphics::Magick
Gtk2::Gdk::Keysyms
HTML::Sanitizer
HTTP::MobileAgent::Plugin::XHTML
ICS
IO::File::String
IkiWiki
Image::ImageShack
Impossible::Requirement::Missing
Irssi
JSON::Client::RPC
JSP
Jungle
Kwiki::Ajax
LEOCHARRE::Class
LSF
Lingua::En::Inflect::Number
Lingua::JA::Heisig
List::Utils
Log::Dispatch::Buffer
MAS::TIFF
MIME::tools
Math::Bigint
Mime::Base64
ModPerl::Const
Mojo::Dom
MojoX::Fixup::XHTML
Mojolicious::Command::generate::nifty_app
Mojolicious::Plugin::Pedro
MooseX::Traits::Attribute::CascadeClear
MooseX::Traits::Attribute::CascadeClearMaster
Music::Tag::Lyrics
Net::FileMaker::XML::ResultSet
Net::OpenDHT
Net::XMPP2
OSSP::uuid
OX::OAuth
Object::By
OpenID::Lite
OpenSSL::Rand
P4
P4::Client
PDL::IO::Storable
POE::Component::Drizzle
POE::Component::GCS::Server
POE::Component::MDBA
POE::Component::PSD::Client
POE::Component::PubSub
POE::Component::Q4M
POE::Component::StackedProcessor
POE::Component::Tie
POE::Session::Cascading
POEx::ProxySession::MessageSender
POEx::PubSub::Event
POEx::Role::SessionInstantiation::Meta::Session::Implementation
POEx::Role::Streaming
POEx::Role::TCPClient
POEx::Role::TCPServer
POEx::Trait::DeferredRegistration
POEx::Trait::ExtraInitialization
POEx::Types
POEx::WorkerPool
Param::Validate
Parse::DebianChangelog
Perl::Critic::Moose
PkgForge
Prima::noX11
Qpsmtpd
Qpsmtpd::Plugin
RRDp
RRDs
Razor::Agent
Razor::Client
Razor::String
ReForm
ReadOnly
Rex::IO::CMDB
Rex::IO::Client
Rex::IO::Server
SOAP::WSDL::XSD::Typelib::Builtin::double
SOAP::WSDL::XSD::Typelib::Builtin::int
SOAP::WSDL::XSD::Typelib::Builtin::long
SOAP::WSDL::XSD::Typelib::Builtin::short
SOAP::WSDL::XSD::Typelib::Builtin::string
SOAP::WSDL::XSD::Typelib::Builtin::unsignedInt
SWF
SWISH::API
SWISH::Filters
Scalar::List::Utils
Senna
Setup::Text::Snippet
Sledge::Exceptions
Sledge::Hammer
Sledge::Pages::Base
Sledge::Template
Sledge::TestPages
Sleepycat::DbXml
Strings::Approx
Sub::Spec
TAP::Archive
TAP::Archive::Harness
Task::Pinto
Task::SDK::Moose
Task::SOSA
Template::Pluign
Template::Toolkit
Test::POD
Test::Require
Test::SimpleUnit
TestParseMIME
Text::Filter
Text::Multi
Text::PrettyTable
Time::Hires
Time::Timestamp
TokyoCabinet
TokyoTyrant
Vcf4_0
VcfReader
VcsTools::File
Video::Info
WWW::Google::SiteMap
WWW::Lipsum
WWW::UsePerl::Journal::Post
WWW::YQL
Win32::API::Prototype
XML::asData
XMLTV
Zeus::ModPerl
distribution_type
generated_by
mojolicious::lexicon
mojolicious::plugin::page::navigator
nginx
override
perl
rt
t::lib::Child1
t::lib::Mother
t::lib::multilog