Created
October 6, 2019 20:07
-
-
Save jonasfj/5a0ab8d3d1477cac4a89c26662a821e3 to your computer and use it in GitHub Desktop.
Count the number of packages on pub.dev that uses package:mono_repo.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Copyright 2019 Google LLC. | |
// SPDX-License-Identifier: Apache-2.0 | |
import 'dart:async' show FutureOr; | |
import 'dart:async'; | |
import 'dart:convert' show utf8, json; | |
import 'dart:io' show Directory, Process, IOException, File; | |
import 'package:http/http.dart' as http; | |
import 'package:retry/retry.dart' show retry; | |
import 'package:pool/pool.dart' show Pool; | |
const _userAgent = 'bot-jonasfj-pub-analysis/1.0 (+jopsen@gmail.com)'; | |
const _pubServer = 'https://pub.dartlang.org'; | |
Future<List<int>> _get(String url) => retry(() async { | |
final response = await http.get(url, headers: { | |
'User-Agent': _userAgent, | |
}).timeout(Duration(minutes: 3)); | |
if (200 >= response.statusCode && response.statusCode < 200) | |
throw Exception('Expected 2xx response, got ${response.statusCode}'); | |
return response.bodyBytes; | |
}, | |
retryIf: (e) => | |
e is IOException || | |
e is TimeoutException || | |
e.toString().contains('Expected 2xx')); | |
dynamic _decodeJson(List<int> data) => json.fuse(utf8).decode(data) as dynamic; | |
Future<List<String>> listAllPackages() async { | |
final body = await _get('$_pubServer/api/packages?compact=1'); | |
return _decodeJson(body)['packages'].cast<String>(); | |
} | |
Future<List<String>> listPackageVersions(String package) async { | |
final body = await _get('$_pubServer/api/packages/$package'); | |
return _decodeJson(body)['versions'] | |
.map<String>((v) => v['version'] as String) | |
.toList(); | |
} | |
Future<String> latestPackageVersion(String package) async { | |
final body = await _get('$_pubServer/api/packages/$package'); | |
return _decodeJson(body)['latest']['version'] as String; | |
} | |
String packageUrl(String package, String version) => | |
'$_pubServer/packages/$package/versions/$version.tar.gz'; | |
Future<void> getPackage(String package, String version, String target) async { | |
final tar = await Process.start('tar', ['zxC', target]); | |
tar.stdin.add(await _get(packageUrl(package, version))); | |
await tar.stdin.close(); | |
if (await tar.exitCode != 0) { | |
throw Exception('Failed to extract downloaded package data'); | |
} | |
} | |
Future<T> withPackage<T>( | |
String package, | |
String version, | |
FutureOr<T> Function(Directory) fn, | |
) async { | |
final temp = await Directory.systemTemp.createTemp('pkg-'); | |
try { | |
await getPackage(package, version, temp.path).timeout(Duration(minutes: 7)); | |
return await fn(temp); | |
} finally { | |
await temp.delete(recursive: true); | |
} | |
} | |
Future<void> main() async { | |
List<String> usesMonoRepo = []; | |
print('# Analyzing all packages'); | |
final allPackages = await listAllPackages(); | |
int failedPackages = 0; | |
final pool = Pool(50, timeout: Duration(minutes: 20)); | |
await Future.wait(allPackages.map((pkg) => pool.withResource(() async { | |
try { | |
await withPackage(pkg, await latestPackageVersion(pkg), (f) async { | |
if (await File.fromUri(f.uri.resolve('mono_pkg.yaml')).exists()) { | |
print('- $pkg contains mono_pkg.yaml'); | |
usesMonoRepo.add(pkg); | |
} else { | |
print('- $pkg checked'); | |
} | |
}); | |
} catch (e) { | |
print('- $pkg failed: $e'); | |
failedPackages++; | |
} | |
}))); | |
print('# Packages using mono_repo:'); | |
usesMonoRepo.forEach((p) => print('- $p')); | |
print('Total number of packages using mono_repo:'); | |
print(' ${usesMonoRepo.length} of ${allPackages.length} packages analyzed'); | |
print(' Failed analysis of $failedPackages packages'); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: count_mono_repo_pub_packages | |
publish_to: none | |
dependencies: | |
http: ^0.12.0 | |
retry: ^3.0.0 | |
pool: ^1.4.0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment