Skip to content

Instantly share code, notes, and snippets.

@jonasfj
Created October 6, 2019 20:07
Show Gist options
  • Save jonasfj/5a0ab8d3d1477cac4a89c26662a821e3 to your computer and use it in GitHub Desktop.
Save jonasfj/5a0ab8d3d1477cac4a89c26662a821e3 to your computer and use it in GitHub Desktop.
Count the number of packages on pub.dev that uses package:mono_repo.
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
import 'dart:async' show FutureOr;
import 'dart:async';
import 'dart:convert' show utf8, json;
import 'dart:io' show Directory, Process, IOException, File;
import 'package:http/http.dart' as http;
import 'package:retry/retry.dart' show retry;
import 'package:pool/pool.dart' show Pool;
const _userAgent = 'bot-jonasfj-pub-analysis/1.0 (+jopsen@gmail.com)';
const _pubServer = 'https://pub.dartlang.org';
Future<List<int>> _get(String url) => retry(() async {
final response = await http.get(url, headers: {
'User-Agent': _userAgent,
}).timeout(Duration(minutes: 3));
if (200 >= response.statusCode && response.statusCode < 200)
throw Exception('Expected 2xx response, got ${response.statusCode}');
return response.bodyBytes;
},
retryIf: (e) =>
e is IOException ||
e is TimeoutException ||
e.toString().contains('Expected 2xx'));
dynamic _decodeJson(List<int> data) => json.fuse(utf8).decode(data) as dynamic;
Future<List<String>> listAllPackages() async {
final body = await _get('$_pubServer/api/packages?compact=1');
return _decodeJson(body)['packages'].cast<String>();
}
Future<List<String>> listPackageVersions(String package) async {
final body = await _get('$_pubServer/api/packages/$package');
return _decodeJson(body)['versions']
.map<String>((v) => v['version'] as String)
.toList();
}
Future<String> latestPackageVersion(String package) async {
final body = await _get('$_pubServer/api/packages/$package');
return _decodeJson(body)['latest']['version'] as String;
}
String packageUrl(String package, String version) =>
'$_pubServer/packages/$package/versions/$version.tar.gz';
Future<void> getPackage(String package, String version, String target) async {
final tar = await Process.start('tar', ['zxC', target]);
tar.stdin.add(await _get(packageUrl(package, version)));
await tar.stdin.close();
if (await tar.exitCode != 0) {
throw Exception('Failed to extract downloaded package data');
}
}
Future<T> withPackage<T>(
String package,
String version,
FutureOr<T> Function(Directory) fn,
) async {
final temp = await Directory.systemTemp.createTemp('pkg-');
try {
await getPackage(package, version, temp.path).timeout(Duration(minutes: 7));
return await fn(temp);
} finally {
await temp.delete(recursive: true);
}
}
Future<void> main() async {
List<String> usesMonoRepo = [];
print('# Analyzing all packages');
final allPackages = await listAllPackages();
int failedPackages = 0;
final pool = Pool(50, timeout: Duration(minutes: 20));
await Future.wait(allPackages.map((pkg) => pool.withResource(() async {
try {
await withPackage(pkg, await latestPackageVersion(pkg), (f) async {
if (await File.fromUri(f.uri.resolve('mono_pkg.yaml')).exists()) {
print('- $pkg contains mono_pkg.yaml');
usesMonoRepo.add(pkg);
} else {
print('- $pkg checked');
}
});
} catch (e) {
print('- $pkg failed: $e');
failedPackages++;
}
})));
print('# Packages using mono_repo:');
usesMonoRepo.forEach((p) => print('- $p'));
print('Total number of packages using mono_repo:');
print(' ${usesMonoRepo.length} of ${allPackages.length} packages analyzed');
print(' Failed analysis of $failedPackages packages');
}
name: count_mono_repo_pub_packages
publish_to: none
dependencies:
http: ^0.12.0
retry: ^3.0.0
pool: ^1.4.0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment