Create a gist now

Instantly share code, notes, and snippets.

perl6 dist.pl6 --user=niner --repo="Inline-Perl5" --branch=master provides Inline::Perl5 lib/Inline/Perl5.pm6
# 2 roles, each provides `meta` (hash like access to META6 structure) and `ioify` which
# gives the Distribution ways to provide an IO::Path/IO::Socket/etc that is ultimately
# an IO object with access to the data that relative path is meant to represent. Both
# return IO::Paths but could easily use .read/.recv
#
# Experimenting with a `files` interface for Distributions to provide the identifiers
# for content that might not be listed in META6 data (likes bin scripts) but would
# be in a manifest
role Distribution::Locally[$path] {
has $!meta;
method meta {
once {
$!meta = try {
my $meta-basename = <META6.json META.info>.first({ $path.IO.child($_).e });
my $meta-path = self.ioify($meta-basename);
%(from-json($meta-path.IO.slurp))
}
}
$!meta;
}
method ioify($relpath) {
IO::Path.new($relpath, :CWD($path));
}
method files($subdir?) {
my @stack = $subdir ?? self.ioify($subdir) !! self.ioify($path);
my $files := gather while ( @stack ) {
my $current = @stack.pop;
take self.ioify($current) if $current.f;
@stack.append( |dir($current) ) if $current.d;
}
}
}
# Works, but commented out to avoid dependency complaints to do a basic test on a path
#
#role Distribution::Remotely::GitHub[:$user = 'ugexe', :$repo = 'zef', :$branch = 'master'] {
# require IO::Socket::SSL;
# use Net::HTTP::GET;
# has $!meta;
#
# method meta{
# $!meta //= try {
# my $json = Net::HTTP::GET("https://raw.githubusercontent.com/$user/$repo/$branch/META6.json").content;
# my $meta = %(from-json($json));
# }
# }
#
# method ioify($relpath) {
# my $content = Net::HTTP::GET("https://raw.githubusercontent.com/{$user}/{$repo}/{$branch}/" ~ $relpath).content;
# my $io = $*TMPDIR.child("{time}-{$*PID}");
# $io.spurt($content);
# $io;
# }
#
# # If we have a local repo this could instead grep the files itself or use `git ls-files`
# # but this naive implementation fetches it from github
# method files($subdir?) {
# my $content = Net::HTTP::GET("https://api.github.com/repos/{$user}/{$repo}/git/trees/{$branch}?recursive=1").content;
# my $json := from-json($content);
# my $files := gather for $json -> $item is copy {
# if !$subdir || $item<path>.starts-with($subdir.subst(/\/$/, '') ~ '/') {
# take $item<path> if $item<type> && $item<type>.lc eq 'blob';
# }
# }
# }
#}
# A Distribution as outlined in S22. In addition to providing `meta` and `ioify` it must also include
# the `content` method (also outlined in S22). This would be for a:
# CompUnit::Repository::Instalation.instal($dist)
# where %scripts, %resources, etc is handles by the $dist. And instead of the CompUnit looking at the META6
# itself and then creating absolute paths (and ultimately IO::Paths) we let the Distribution handle that
# (as in the 2 roles above). CompUnit::Repository::Installation only needs to access data via the .content API:
# # RESOURCES:
# $dist.content(<resources configuration config.json>).?spurt($save-as)
# # we can handle certain subkeys like <resources><library> differently (do platform lib naming)
# $dist.content(<resources library somelib>).?spurt($save-as)
# MODULES/PROVIDES:
# $dist.content(<provides Zef::App lib/Zef/App.pm6>).?spurt($save-as)
# After the CompUnit::R spurts each file it can do any precomp or anything else it wants using its (now local) copy
use nqp;
class Distribution::Installation is Distribution {
proto method content(|) {*}
multi method content('provides', *@keys) {
# provides specific parsing
my $relpath = ~@keys.tail;
my $io = self.?ioify($relpath) // $relpath.IO;
$io;
}
multi method content('resources', 'libraries', *@keys) {
my $relpath = self!make-path: 'resources', 'libraries', |@keys.reduce: { $^a.IO.child($^b) };
my $abspath = self.?ioify($relpath) // $relpath.IO;
my $lib = $*VM.platform-library-name(self.?absolutify($relpath) // $relpath.IO);
$lib;
}
multi method content('resources', *@keys) {
my $relpath = self!make-path: 'resources', |@keys.reduce: { $^a.IO.child($^b) };
my $io = self.?ioify($relpath) // $relpath.IO;
$io;
}
multi method content('bin') {
# META6 has no where to declare bin/ scripts so this specific method is for returning
# the files actually found in the bin/ directory
@ = self.files('bin');
}
multi method content('bin', *@keys) {
# In case the
# @keys[0] = @keys[0].subst(/^bin\//, '');
my $relpath = self!make-path: 'bin', |@keys.reduce: { $^a.IO.child($^b) };
my $io = self.?ioify($relpath) // $relpath.IO;
$io;
}
method !make-path(*@parts) {
@parts.reduce({ $^a.IO.child($^b) })
}
method Str() {
"{$.meta<name>}:ver<{$.meta<ver> // $.meta<version> // ''}>:auth<{$.meta<auth> // ''}>:api<{$.meta<api> // ''}>";
}
method id() {
nqp::sha1(self.Str);
}
}
# *@contents should match S22 spec, so access the keys of the META6 as specified
# i.e. <provides Zef lib/Zef.pm6> gives us an IO that can be read/slurp'd (like in a CompUnit::Repository)
# giving no keys will give the META6.json
#
# perl6 dist.pl6 --user=niner --repo="Inline-Perl5" --branch=master provides Inline::Perl5 lib/Inline/Perl5.pm6
# perl6 dist.pl6 --path=/home/perl6/Inline-Perl5 provides Inline::Perl5 lib/Inline/Perl5.pm6
#| `--user=* --repo=* --branch=master` Access from github remotely
multi sub MAIN(:$user = 'ugexe', :$repo = 'zef', :$branch = 'master', *@contents) {
my $dist = Distribution::Installation.new() but Distribution::Remotely::GitHub[:$user, :$repo, :$branch];
say @contents.elems ?? $dist.content(|@contents).slurp !! $dist.meta;
}
#| `--path=.` Access from a local dist
multi sub MAIN(:$path, *@contents) {
my $dist = Distribution::Installation.new() but Distribution::Locally[$path];
say @contents.elems ?? $dist.content(|@contents).slurp !! $dist.meta;
}
#| `--install-to` try to install a dist via the CU::R::I above
multi sub MAIN(:$install-path) {
my $dist = Distribution::Installation.new() but Distribution::Locally[$path];
my $prefix = ~CompUnit::RepositoryRegistry.repository-for-name('site').prefix;
CompUnit::Repository::Installation2.new(:$prefix).install($dist);
# A semi-tested, hypothetical change to CU::R::I to use the proposed Distribution access methods .meta and .content
# as used in the accompanying `dist.pl6` (Distribution::Locally, Distribution::Remotely::GitHub). Seems to work
# up until precompilation (presumably because this CU::R is not linked to the $*REPOS chain that happen before late load)
class CompUnit::Repository::Installation2 does CompUnit::Repository::Locally does CompUnit::Repository::Installable {
has $!cver = nqp::hllize(nqp::atkey(nqp::gethllsym('perl6', '$COMPILER_CONFIG'), 'version'));
has %!loaded;
has $!precomp;
has $!id;
my $verbose := nqp::getenvhash<RAKUDO_LOG_PRECOMP>;
submethod BUILD(:$!prefix, :$!lock, :$!WHICH, :$!next-repo) { }
method writeable-path {
$.prefix.w ?? $.prefix !! IO::Path;
}
method !writeable-path {
self.can-install ?? $.prefix !! IO::Path;
}
method can-install() {
$.prefix.w || ?(!$.prefix.e && try { $.prefix.mkdir } && $.prefix.e);
}
my $windows_wrapper = '@rem = \'--*-Perl-*--
@echo off
if "%OS%" == "Windows_NT" goto WinNT
#perl# "%~dpn0" %1 %2 %3 %4 %5 %6 %7 %8 %9
goto endofperl
:WinNT
#perl# "%~dpn0" %*
if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto endofperl
if %errorlevel% == 9009 echo You do not have Perl in your PATH.
if errorlevel 1 goto script_failed_so_exit_with_non_zero_val 2>nul
goto endofperl
@rem \';
__END__
:endofperl
';
my $perl_wrapper = '#!/usr/bin/env #perl#
sub MAIN(:$name is copy, :$auth, :$ver, *@, *%) {
shift @*ARGS if $name;
shift @*ARGS if $auth;
shift @*ARGS if $ver;
$name //= \'#dist-name#\';
my @installations = $*REPO.repo-chain.grep(CompUnit::Repository::Installable);
my @binaries = flat @installations.map: { .files(\'bin/#name#\', :$name, :$auth, :$ver) };
unless +@binaries {
@binaries = flat @installations.map: { .files(\'bin/#name#\') };
if +@binaries {
note q:to/SORRY/;
===SORRY!===
No candidate found for \'#name#\' that match your criteria.
Did you perhaps mean one of these?
SORRY
my %caps = :name([\'Distribution\', 12]), :auth([\'Author(ity)\', 11]), :ver([\'Version\', 7]);
for @binaries -> $dist {
for %caps.kv -> $caption, @opts {
@opts[1] = max @opts[1], ($dist{$caption} // \'\').Str.chars
}
}
note \' \' ~ %caps.values.map({ sprintf(\'%-*s\', .[1], .[0]) }).join(\' | \');
for @binaries -> $dist {
note \' \' ~ %caps.kv.map( -> $k, $v { sprintf(\'%-*s\', $v.[1], $dist{$k} // \'\') } ).join(\' | \')
}
}
else {
note "===SORRY!===\nNo candidate found for \'#name#\'.\n";
}
exit 1;
}
exit run($*EXECUTABLE-NAME, @binaries[0].hash.<files><bin/#name#>, @*ARGS).exitcode
}';
method !sources-dir() {
my $sources = $.prefix.child('sources');
$sources.mkdir unless $sources.e;
$sources
}
method !resources-dir() {
my $resources = $.prefix.child('resources');
$resources.mkdir unless $resources.e;
$resources
}
method !dist-dir() {
my $dist = $.prefix.child('dist');
$dist.mkdir unless $dist.e;
$dist
}
method !bin-dir() {
my $bin = $.prefix.child('bin');
$bin.mkdir unless $bin.e;
$bin
}
method !add-short-name($dist) {
my $short-dir = $.prefix.child('short');
$short-dir.mkdir unless $short-dir.e;
my $id = nqp::sha1($dist.meta<name>);
my $lookup = $short-dir.child($id).open(:a);
$lookup.say: $dist.id;
$lookup.close;
}
method !file-id(Str $name, Str $dist-id) {
my $id = $name ~ $dist-id;
nqp::sha1($id)
}
# The previous `%sources, %scripts?, %resources?` should not be neccesary for `Installable`
# Maybe a more custom CU::R, but I see CU::R::I as giving an easy interface to doing the more common
# things (of what I think is envisioned for external CompUnit::Repositorys) to Distribution via
# a interface injection approach.
# Summary:
# - `Distribution` should be seen as having no ties to if something is actually installed or not.
# Instead it represents how to access the data of any type of Distribution that uses the
# new interface that supplies the `.content` method.
# - CU::Rs (CompUnit::Repository::*) may or may not choose to accept this type of Distribution
# for its optional `.install` but the important bit is once the CU::R gets all the data
# it needs through the Distribution (think: slurping all source code) it can save it
# as a different Distribution it knows how to load later (ex: `CU::R::Whatever::Distribution`)
# and should no longer be thought of as a Distribution at all (really it is, but to the CU::R
# it only needs to consider it as a CompUnit)
method install(Distribution $dist, :$force = True) {
$!lock.protect( {
my @*MODULES;
my $path = self!writeable-path or die "No writeable path found, $.prefix not writeable";
my $lock //= $.prefix.child('repo.lock').open(:create, :w);
$lock.lock(2);
# Distribution.Str() (and by extension Distribution.id() used below) need improvement.
# Example: a META6 that contains no `auth` and an `author` field with value `user <a@b.org>`
# Problem: Distribution.Str() would give us `Some::Module:auth<user<@b.org>>`
# Explanation: I would want .Str() on a Distribution to give an invokable identity
# `use Some::Module:auth<user <@b.org>>;` # does not work
# `use Some::Module:auth('user <@b.org>');` # does work
# Solutions:
# 1) .Str() could use the format in the 2nd explanation example
# a) maybe it should be .perl/.identity method instead?
# b) is just a stop gap solution that seems to work now and easily replaced later
# 2) - Change Distribution's auth method to:
# `method auth { $!auth // ($!owner && $!cs ?? "{$!owner}:{$!cs}" !! '') }
# a) `cs` is short for content-storage
# b) content storage and $!owner are both mentioned in s22
# c) s22 seems to imply the current `authority` field is what `content storage` is meant to replace
# - Average author would use `auth` field, but could use `owner` and `cs` (or `authority`?)
# fields to generate the implied auth if it cant explicitly find an `auth` field
# - This all assumes the META6.json `version`/`owner`/`auth` is not meant to contain
# ranges, only explicit values. Ranges should be used when searching for an explicit
# identity, not for declaring one's identity (at least in META6.json)
# 3) Both 1 and 2
my $dist-id = $dist.id;
my $dist-dir = self!dist-dir;
if not $force and $dist-dir.child($dist-id) ~~ :e {
$lock.unlock;
fail "$dist already installed";
}
my $sources-dir = self!sources-dir;
my $resources-dir = self!resources-dir;
my $bin-dir = self!bin-dir;
my $is-win = Rakudo::Internals.IS-WIN;
self!add-short-name($dist); # so scripts can find their dist
# XXX: :time is clearly busted for something like a socket
my %sources = gather for $dist.meta<provides>.kv -> $name, $relpath {
my $id = self!file-id($relpath, $dist-id);
my $destination = $sources-dir.child($id);
note("Installing {$name} for {$dist.meta<name>}") if $verbose and $name ne $dist.meta<name>;
$destination.spurt: $dist.content('provides', $relpath).slurp;
take ($name => {
pm => {
:file(self!file-id($relpath, $dist-id)),
:time(try $dist.ioify($relpath).modified.Num),
:$!cver
}
});
}
my %scripts = gather for $dist.files('bin') -> $file is copy {
my $id = self!file-id($file.relative, $dist-id);
my $destination = $resources-dir.child($id);
my $withoutext = $file.relative.subst(/\.[exe|bat]$/, '');
for '', '-j', '-m' -> $be {
"$path/$withoutext$be".IO.spurt:
$perl_wrapper\
.subst('#name#', $file.IO.basename, :g)
.subst('#perl#', "perl6$be")\
.subst('#dist-name#', $dist.meta<name>);
if $is-win {
"$path/$withoutext$be.bat".IO.spurt:
$windows_wrapper.subst('#perl#', "perl6$be", :g);
}
else {
"$path/$withoutext$be".IO.chmod(0o755);
}
}
self!add-short-name($dist);
take $file => $id;
$destination.spurt: $dist.content('bin', $file.relative($dist.ioify('bin'))).slurp;
}
my %resources = gather for $dist.meta<resources>.kv -> $name, $relpath is copy {
my $id = self!file-id($relpath, $dist-id) ~ '.' ~ $relpath.IO.extension;
my $destination = $resources-dir.child($id);
take $name => $id;
$destination.spurt: $dist.content('resources', $relpath).slurp;
}
my %meta = $dist.meta;
%meta<provides> = %sources;
%meta<files> = |%scripts, |%resources;
$dist-dir.child($dist-id).spurt: to-json(%meta);
my $precomp = $*REPO.precomp-repository;
my $*RESOURCES = Distribution::Resources.new(:repo(self), :$dist-id);
my %done;
for %sources.values.map(*.values[0]<file>) -> $id {
my $source = $sources-dir.child($id);
if $precomp.may-precomp {
my $rev-deps-file = ($precomp.store.path($*PERL.compiler.id, $id) ~ '.rev-deps').IO;
my @rev-deps = $rev-deps-file.e ?? $rev-deps-file.lines !! ();
if %done{$id} { note "(Already did $id)" if $verbose; next }
note("Precompiling $id") if $verbose;
$precomp.precompile($source.IO, $id, :force);
%done{$id} = 1;
for @rev-deps -> $rev-dep-id {
if %done{$rev-dep-id} { note "(Already did $rev-dep-id)" if $verbose; next }
note("Precompiling $rev-dep-id") if $verbose;
my $source = $sources-dir.child($rev-dep-id);
$precomp.precompile($source, $rev-dep-id, :force) if $source.e;
%done{$rev-dep-id} = 1;
}
}
}
# reset cached id so it's generated again on next access.
# identity changes with every installation of a dist.
$!id = Any;
$lock.unlock;
} ) }
method files($file, :$name, :$auth, :$ver) {
my @candi;
my $prefix = self.prefix;
my $lookup = $prefix.child('short').child(nqp::sha1($name));
if $lookup.e {
my $dist-dir = self!dist-dir;
for $lookup.lines -> $dist-id {
my $dist = from-json($dist-dir.child($dist-id).slurp);
my $dver = $dist<ver>
?? nqp::istype($dist<ver>,Version)
?? $dist<ver>
!! Version.new( $dist<ver> )
!! Version.new('0');
if (!$name || $dist<name> ~~ $name)
&& (!$auth || $dist<auth> ~~ $auth)
&& (!$ver || $dver ~~ $ver) {
with $dist<files>{$file} {
my $candi = %$dist;
$candi<ver> = $dver;
$candi<files>{$file} = $prefix.abspath ~ '/resources/' ~ $candi<files>{$file}
unless $candi<files>{$file} ~~ /^$prefix/;
@candi.push: $candi;
}
}
}
}
@candi
}
method need(
CompUnit::DependencySpecification $spec,
CompUnit::PrecompilationRepository $precomp = self.precomp-repository(),
)
returns CompUnit:D
{
if $spec.from eq 'Perl6' {
my $lookup = $.prefix.child('short').child(nqp::sha1($spec.short-name));
if $lookup.e {
my $dist-dir = self!dist-dir;
my @dists = $lookup.lines.unique.map({
$_ => from-json($dist-dir.child($_).slurp)
}).grep({
$_.value<auth> ~~ $spec.auth-matcher
and Version.new(~$_.value<ver> || '0') ~~ $spec.version-matcher
and $_.value<provides>{$spec.short-name}:exists
});
for @dists.sort(*.value<ver>).reverse.map(*.kv) -> ($dist-id, $dist) {
return %!loaded{$spec.short-name} if %!loaded{$spec.short-name}:exists;
my $dver = $dist<ver>
?? nqp::istype($dist<ver>,Version)
?? $dist<ver>
!! Version.new( ~$dist<ver> )
!! Version.new('0');
my $loader = $.prefix.child('sources').child(
$dist<provides>{$spec.short-name}<pm pm6>.first(*.so)<file>
);
my $*RESOURCES = Distribution::Resources.new(:repo(self), :$dist-id);
my $handle;
my $id = $loader.basename;
if $precomp.may-precomp {
$handle = (
$precomp.load($id, :since($loader.modified)) # already precompiled?
or $precomp.precompile($loader, $id) and $precomp.load($id) # if not do it now
);
if $*W and $*W.is_precompilation_mode {
if $handle {
say "$id $loader";
}
else {
nqp::exit(0);
}
}
}
my $precompiled = defined $handle;
$handle //= CompUnit::Loader.load-source-file($loader);
my $compunit = CompUnit.new(
:$handle,
:short-name($spec.short-name),
:version($dver),
:auth($dist<auth> // Str),
:repo(self),
:repo-id($id),
:$precompiled,
:distribution(Distribution.new(|$dist)),
);
return %!loaded{$compunit.short-name} = $compunit;
}
}
}
return self.next-repo.need($spec, $precomp) if self.next-repo;
X::CompUnit::UnsatisfiedDependency.new(:specification($spec)).throw;
}
method resource($dist-id, $key) {
my $dist = from-json(self!dist-dir.child($dist-id).slurp);
self!resources-dir.child($dist<files>{$key})
}
method id() {
return $!id if $!id;
$!id = self.CompUnit::Repository::Locally::id();
my $dist-dir = $.prefix.child('dist');
$!id = nqp::sha1($!id ~ ($dist-dir.e ?? $dist-dir.dir !! ''))
}
method short-id() { 'inst' }
method loaded() returns Iterable {
return %!loaded.values;
}
method precomp-repository() returns CompUnit::PrecompilationRepository {
$!precomp := CompUnit::PrecompilationRepository::Default.new(
:store(
CompUnit::PrecompilationStore::File.new(
:prefix(self.prefix.child('precomp')),
)
),
) unless $!precomp;
$!precomp
}
sub provides-warning($is-win, $name) {
my ($red,$clear) = Rakudo::Internals.error-rcgye;
note "$red==={$clear}WARNING!$red===$clear
The distribution $name does not seem to have a \"provides\" section in its META.info file,
and so the packages will not be installed in the correct location.
Please ask the author to add a \"provides\" section, mapping every exposed namespace to a
file location in the distribution.
See http://design.perl6.org/S22.html#provides for more information.\n";
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment