pkgcrap/pkgcrap/outdated_check.py

import urllib.request
import pkgcrap.parse as parse
import json
from pkgcrap.util import blacklisted_tags_get, maintained_packages_get
from functools import cmp_to_key
from libversion import version_compare

def guess_repo(uri):
    forges = [
        ('git', 'https://github.com', True),
        ('git', 'https://gitlab.com', True),
        ('git', 'https://git.sr.ht', False),
        ('pypi', 'mirror://pypi', False),
    ]
    for forge in forges:
        if uri.startswith(forge[1]):
            if forge[0] == 'git':
                suffix = '.git' if forge[2] == True else ''
                return (forge[0], '/'.join(uri.split('/')[:5])+suffix)
            if forge[0] == 'pypi':
                return (forge[0], uri.split('/')[4])
    return None

def repos_from_metadata(pkg):
    remotes = []
    for remote in pkg.metadata.remotes:
        if remote[0] == 'github':
            remotes.append(('git', 'https://github.com/'+remote[1]+'.git'))
        if remote[0] == 'gitlab':
            remotes.append(('git', 'https://gitlab.com/'+remote[1]+'.git'))
        if remote[0] == 'pypi':
            remotes.append(('pypi', remote[1]))
    return remotes

def repo_from_repo_uri(pkg):
    if 'EGIT_REPO_URI' in pkg.ebuilds[0].vars:
        return ('git', pkg.ebuilds[0].vars['EGIT_REPO_URI'].replace('${PN}', pkg.name))
    if 'EHG_REPO_URI' in pkg.ebuilds[0].vars:
        return ('hg', pkg.ebuilds[0].vars['EHG_REPO_URI'].replace('${PN}', pkg.name))

def repo_from_src_uri(pkg):
    for eb in pkg.ebuilds:
        if 'SRC_URI' in eb.vars:
            return guess_repo(eb.vars['SRC_URI'].split(' ')[0].replace('${PN}', pkg.name))

def repo_from_homepage(pkg):
    for eb in pkg.ebuilds:
        if 'HOMEPAGE' in eb.vars:
            return guess_repo(eb.vars['HOMEPAGE'])

def repos_from_pkg(pkg):
    repos = []
    repos += repos_from_metadata(pkg)
    repos.append(repo_from_repo_uri(pkg))
    repos.append(repo_from_src_uri(pkg))
    repos.append(repo_from_homepage(pkg))
    return [repo for repo in repos if repo != None]

def repo_get_latest_git(uri, pkg, verbose):
    global blacklisted_tags
    try:
        r = urllib.request.urlopen(uri+'/info/refs?service=git-upload-pack')
    except urllib.error.HTTPError:
        if verbose:
            print(pkg.full_name+':', 'Invalid repo!', uri)
        return None
    tags = []
    for line in reversed(r.read().decode('utf-8').split("\n")):
        if line[4:44] in blacklisted_tags:
            continue
        line = line.split(' ')[-1].split("\t")[-1]
        if line[:10] == 'refs/tags/' and line[-3:] != '^{}':
            tag = line[10:]
            if tag in blacklisted_tags:
                continue
            if tag.startswith(pkg.name):
                tag = tag[len(pkg.name):]
            version_prefixes = ['version', 'release']
            for prefix in version_prefixes:
                if tag.startswith(prefix):
                    tag = tag[len(prefix):]
                    break
            if not tag[0].isdigit() and tag[1].isdigit():
                tag = tag[1:]
            tags.append(tag)
    if len(tags) < 1:
        return None
    latest = sorted(tags, key=cmp_to_key(version_compare))[-1]
    if not latest[0].isdigit():
        return None
    return latest

def repo_get_latest_hg(uri, pkg, verbose):
    global blacklisted_tags
    forges = [
        ('https://hg.sr.ht/', '/raw/'),
    ]
    raw_path = '/raw-file/tip/'
    for forge in forges:
        if uri.startswith(forge[0]):
            raw_path = forge[1]
    try:
        r = urllib.request.urlopen(uri+raw_path+'.hgtags')
    except urllib.error.HTTPError:
        if verbose:
            print(pkg.full_name+':', 'Invalid repo!', uri)
        return None
    for line in r.read().decode('utf-8').split("\n"):
        tag = line[41:]
        if line[:40] in blacklisted_tags or tag in blacklisted_tags:
            continue
        latest = tag
    return latest

def repo_get_latest_pypi(uri, pkg, verbose):
    try:
        r = urllib.request.urlopen('https://pypi.org/pypi/'+uri+'/json')
    except urllib.error.HTTPError:
        if verbose:
            print(pkg.full_name+':', 'Invalid PyPI package!', uri)
        return None
    versions = list(json.load(r)['releases'].keys())
    latest = sorted(versions, key=cmp_to_key(version_compare))[-1]
    return latest

def repo_get_latest(repo, pkg, verbose):
    if repo[0] == 'git':
        return repo_get_latest_git(repo[1], pkg, verbose)
    if repo[0] == 'hg':
        return repo_get_latest_hg(repo[1], pkg, verbose)
    if repo[0] == 'pypi':
        return repo_get_latest_pypi(repo[1], pkg, verbose)
    return None

def outdated_check(pkg, verbose=False):
    if len(pkg.ebuilds) == 1 and pkg.ebuilds[0].live:
        if verbose:
            print(pkg.full_name+':', 'Only has live ebuild')
        return
    repos = repos_from_pkg(pkg)
    if len(repos) == 0:
        if verbose:
            print(pkg.full_name+':', 'Repo not found')
        return
    current = pkg.version_latest()
    for repo in repos:
        if verbose:
            print(pkg.full_name+':', 'Checking '+repo[0]+' repo ('+repo[1]+')')
        latest = repo_get_latest(repo, pkg, verbose)
        if latest != None:
            break
    if latest == None:
        if verbose:
            print(pkg.full_name+':', 'Unable to find latest version!')
    else:
        if version_compare(current, latest) == -1:
            print(pkg.full_name+':', 'Outdated package!', current, '->', latest)
        elif verbose:
            print(pkg.full_name+':', 'Up to date!', current, '->', latest)

def main(args):
    if 'all' in args:
        packages = maintained_packages_get()
        verbose = 'verbose' in args
    else:
        packages = [parse.package.from_path('.')]
        verbose = True
    global blacklisted_tags
    blacklisted_tags = blacklisted_tags_get()
    for pkg in packages:
        pkg.load()
        outdated_check(pkg, verbose=verbose)