Endless script to check appstream files for common appdata invalidity

appstream-validator.py

#!/usr/bin/env python3

# This script parses all the local appstream data
# to check for apps that are missing important metadata
# that is required for proper display in gnome-software.
#
# It is recommended to run `flatpak update --apstream`
# prior to this script.

import xml.etree.ElementTree as ET
import gzip
import os
import platform
import sys

arch = platform.machine()
if arch.startswith('arm'):
    arch = 'arm'

single_disk_appstream_dir = '/var/lib/flatpak/appstream'
split_disk_appstream_dir = '/var/endless-extra/flatpak/appstream'

if os.path.isdir(single_disk_appstream_dir):
    appstream_dir = single_disk_appstream_dir
elif os.path.isdir(split_disk_appstream_dir):
    appstream_dir = split_disk_appstream_dir
else:
    print('No appstream at %s or %s' %
          (single_disk_appstream_dir, split_disk_appstream_dir))
    sys.exit()

# These are the official top-level freedesktop.org categories,
# plus "Reference", which has been added as a top-level category
# in gnome-software
supported_categories = [
    'AudioVideo',
    'Development',
    'Education',
    'Game',
    'Graphics',
    'Network',
    'Office',
    'Reference',
    'Science',
    'Settings',
    'System',
    'Utility'
    ]

total_apps = {}
invalid_apps = {}
duplicate_apps = {}

remotes = sorted(os.listdir(appstream_dir))
for remote in remotes:
    total_apps[remote] = 0
    invalid_apps[remote] = 0
    duplicate_apps[remote] = 0

    appstream_gz = os.path.join(appstream_dir, remote, arch, 'active',
                                'appstream.xml.gz')
    if not os.path.isfile(appstream_gz):
        continue

    with gzip.open(appstream_gz, 'r') as f:
        appstream = f.read()

    root = ET.fromstring(appstream)

    prev_id = None

    components = root.findall('component')
    for component in sorted(components, key=lambda x: x.findtext('id')):

        # Only look at desktop apps, not runtimes
        flatpak_type = component.get('type')
        if flatpak_type != 'desktop':
            continue

        # For now, until we remove all the old eos-apps eos3.x
        # branches, we want to filter them out by only looking
        # at the eos3 branch for all remotes other than flathub
        bundle = component.findtext('bundle')
        if not (remote == 'flathub' or bundle.endswith('eos3')):
            continue

        # Ignore any apps that are vetoed for no display
        vetos = component.find('vetos')
        if vetos:
            no_display = False
            for veto in vetos.findall('veto'):
                if veto.text == 'NoDisplay=true':
                    no_display = True
                    break
            if no_display:
                continue

        total_apps[remote] += 1
        invalid = False

        # Detect duplicates with and without .desktop suffix
        # Since apps are sorted alphabetically by ID, we only
        # need to check the previous ID
        app_id = component.findtext('id')
        if app_id.endswith('.desktop'):
            if app_id[:-len('.desktop')] == prev_id:
                print('%s: %s is a duplicate of %s' % (remote, app_id, prev_id))
                duplicate_apps[remote] += 1
                total_apps[remote] -= 1
        prev_id = app_id

        # Check that the app has a name
        if not component.findtext('name'):
            invalid = True
            print('%s: %s has no name' % (remote, app_id))

        # Check that the app has a summary
        if not component.findtext('summary'):
            invalid = True
            print('%s: %s has no summary' % (remote, app_id))

        # Check that the app has a description
        if not component.find('description'):
            invalid = True
            print('%s: %s has no description' % (remote, app_id))

        # Check that the app has at least one supported category
        categories = component.find('categories')
        if categories:
            supported = False
            for category in categories.findall('category'):
                if category.text in supported_categories:
                    supported = True
                    break
            if not supported:
                invalid = True
                print('%s: %s has no supported category' % (remote, app_id))
        else:
            invalid = True
            print('%s: %s has no category' % (remote, app_id))

        # Check that the app has an icon
        if not component.findall('icon'):
            invalid = True
            print('%s: %s has no icon' % (remote, app_id))

        # Check that the app has screenshots
        if not component.find('screenshots'):
            invalid = True
            print('%s: %s has no screenshots' % (remote, app_id))

        if invalid:
            invalid_apps[remote] += 1

print()
print('Summary')
print('=======')
for remote in remotes:
    print('%s: total apps = %d, invalid = %d, duplicate = %d' %
          (remote, total_apps[remote], invalid_apps[remote],
duplicate_apps[remote]))