honesty-fixes.diff

diff --git a/honesty/cache.py b/honesty/cache.py
index 2f0abd8..bdd1829 100644
--- a/honesty/cache.py
+++ b/honesty/cache.py
@@ -4,6 +4,7 @@ Cache-related stuff.
 
 import asyncio
 import os
+import sys
 import posixpath
 import urllib.parse
 from pathlib import Path
@@ -80,6 +81,8 @@ class Cache:
         Returns a Path for where the cache wanted to save it.  We make effort to
         be concurrent-safe (last one wins).
         """
+        sys.stderr.write(f"fetch {pkg}                      \r")
+        sys.stderr.flush()
 
         # Because parse_index doesn't understand entities, there are some urls
         # that we currently get that we shouldn't bother fetching.
diff --git a/honesty/cmdline.py b/honesty/cmdline.py
index 3797ba8..eb46c49 100644
--- a/honesty/cmdline.py
+++ b/honesty/cmdline.py
@@ -245,6 +245,7 @@ async def download(
 @click.option(
     "--index-url", help="Alternate index url (uses HONESTY_INDEX_URL or pypi by default"
 )
+@click.option("--wheel-ok", is_flag=True, type=bool)
 @click.argument("package_names", nargs=-1)
 @wrap_async
 async def extract(
@@ -254,6 +255,7 @@ async def extract(
     dest: str,
     index_url: Optional[str],
     package_names: List[str],
+    wheel_ok: bool,
 ) -> None:
     if dest and len(package_names) > 1:
         # select_versions() may also result in more than one, but that seems
@@ -278,6 +280,8 @@ async def extract(
 
             rel = package.releases[selected_versions[0]]
             sdists = [f for f in rel.files if f.file_type == FileType.SDIST]
+            if not sdists and wheel_ok:
+                sdists = rel.files  # TODO: Hacky, ought to have a prio
             if not sdists:
                 raise click.ClickException(f"{package.name} no sdists")
 
@@ -373,7 +377,7 @@ multiple times (with different versions).
 )
 @click.option("--sys-platform", default="linux", help="linux,darwin,win32")
 @click.option("--historical", help="yyyy-mm-dd of a historical date to simulate")
-@click.option("--use-json", is_flag=True, default=True)
+@click.option("--use-json", is_flag=True, default=False)
 @click.argument("package_name")
 def deps(
     include_extras: bool,
@@ -436,11 +440,11 @@ def select_versions(
 
     if selector == "":
         # latest; we have a function called `list`
-        version = [x for x in package.releases.keys()][-1]
+        version = [x for x in package.releases.keys() if not package.releases[x].yanked][-1]
         return [version]
     elif selector == "*":
         # we have a function called `list`
-        return [x for x in package.releases.keys()]
+        return [x for x in package.releases.keys() if not package.releases[x].yanked]
     else:
         pv = parse_version(selector)
         if pv not in package.releases:
diff --git a/honesty/deps.py b/honesty/deps.py
index 07f0c19..632aef9 100644
--- a/honesty/deps.py
+++ b/honesty/deps.py
@@ -11,7 +11,7 @@ from zipfile import ZipFile
 import click
 from packaging.markers import Marker
 from packaging.requirements import Requirement
-from packaging.specifiers import SpecifierSet
+from packaging.specifiers import SpecifierSet, InvalidSpecifier
 from pkginfo.distribution import parse as distribution_parse
 from pkginfo.wheel import Wheel
 from seekablehttpfile import SeekableHttpFile
@@ -103,7 +103,7 @@ class DepWalker:
         self.only_first = only_first
         self.trim_newer = trim_newer
 
-    def walk(self, include_extras: bool, use_json: bool = True) -> DepNode:
+    def walk(self, include_extras: bool, use_json: bool = False) -> DepNode:
         with Cache(fresh_index=True) as cache:
             while self.queue:
                 parent, item = self.queue.pop(0)
@@ -379,18 +379,22 @@ def _find_compatible_version(
             if oldest_file is not None and oldest_file > trim_newer:
                 continue
 
-        # requires_python is set on FileEntry, not PackageRelease
-        # arbitrarily take the first one.
-        requires_python = None
-        for fe in v.files:
-            if fe.requires_python:
-                requires_python = SpecifierSet(fe.requires_python)
-                break
+        try:
+            # requires_python is set on FileEntry, not PackageRelease
+            # arbitrarily take the first one.
+            requires_python = None
+            for fe in v.files:
+                if fe.requires_python:
+                    requires_python = SpecifierSet(fe.requires_python)
+                    break
 
-        # LOG.debug(f"CHECK {package.name} {python_version} against {requires_python}: {k}")
-        if not requires_python or python_version in requires_python:
-            LOG.debug("  include %s", k)
-            possible.append(k)
+            # LOG.debug(f"CHECK {package.name} {python_version} against {requires_python}: {k}")
+            if not requires_python or python_version in requires_python:
+                LOG.debug("  include %s", k)
+                possible.append(k)
+        except InvalidSpecifier as e:
+            LOG.debug("  bad specifier: {e!r}")
+            pass
 
     if not possible:
         raise ValueError(f"{package.name} incompatible with {python_version}")
@@ -466,8 +470,11 @@ def print_deps(
                 f"{prefix}{x.target.name}{dep_extras} (=={x.target.version}) (already listed){' ; ' + str(x.markers) if x.markers else ''}"
             )
         else:
+            if any(x[0] == key[0] for x in seen):
+                color = "magenta"
+            else:
+                color = "red" if not x.target.has_sdist else "green"
             seen.add(key)
-            color = "red" if not x.target.has_sdist else "green"
             click.echo(
                 prefix
                 + click.style(
diff --git a/honesty/releases.py b/honesty/releases.py
index bfd00bb..6087c73 100644
--- a/honesty/releases.py
+++ b/honesty/releases.py
@@ -167,6 +167,7 @@ class PackageRelease:
     parsed_version: Version
     files: List[FileEntry]
     requires: Optional[List[str]] = None
+    yanked: Optional[str] = None
 
 
 @dataclass
@@ -260,6 +261,7 @@ async def async_parse_index(
             except InvalidVersion:
                 continue
             if pv not in releases:
+                # TODO yanked
                 releases[pv] = PackageRelease(version=v, parsed_version=pv, files=[])
             releases[pv].files.append(fe)
     else:
@@ -278,10 +280,16 @@ async def async_parse_index(
                 # up in the simple index either.
                 continue
             pv = parse_version(k)
-            releases[pv] = PackageRelease(version=k, parsed_version=pv, files=[])
+            releases[pv] = PackageRelease(version=k, parsed_version=pv, files=[], yanked="default")
             for release_file in release:
                 try:
                     releases[pv].files.append(FileEntry.from_json(k, release_file))
+                    # If all files are yanked, then the release is considered
+                    # yanked.  Pick the reason arbitrarily.
+                    if release_file.get("yanked"):
+                        releases[pv].yanked = release_file.get("yanked_reason")
+                    else:
+                        releases[pv].yanked = None
                 except UnexpectedFilename:
                     if strict:
                         raise