From da7f321e9338624375eab758d822445d2846e771 Mon Sep 17 00:00:00 2001 From: hseg Date: Tue, 2 Mar 2021 22:47:44 +0200 Subject: [PATCH] Fix packaging bugs (#129) * Autogenerate `AUTHORS` * Fix `setup.py` using wrong completion files * Complete `ChangeLog` -> `Changelog.md` rename * Make `make tar` respect DESTDIR * Remove `bin/` `yt-dlp` and `docs/` from tar and sdist * Make `pypi-files` build all files needed for `python setup.py` * Add `completions` alias * Add `devscripts/` and `supportedsites.md` to pip sdist * Remove `man` target * Remove `README.txt` from sdist * Make `clean` more granular * Move aliases to top Authored by: hseg --- .gitignore | 2 ++ MANIFEST.in | 12 ++++----- Makefile | 58 ++++++++++++++++++++++++------------------- devscripts/release.sh | 10 ++++---- setup.py | 8 +++--- 5 files changed, 49 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 7a2d92d7b..af0259f84 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,9 @@ updates_key.pem *.class # Generated +AUTHORS README.txt +.mailmap *.1 *.bash-completion *.fish diff --git a/MANIFEST.in b/MANIFEST.in index 480b5c700..38d83a9a5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,9 @@ -include README.md -include LICENSE include AUTHORS -include ChangeLog -include yt-dlp.bash-completion -include yt-dlp.fish +include Changelog.md +include LICENSE +include README.md +include completions/*/* +include supportedsites.md include yt-dlp.1 -recursive-include docs Makefile conf.py *.rst +recursive-include devscripts * recursive-include test * diff --git a/Makefile b/Makefile index c3669a036..e55f11103 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,28 @@ all: yt-dlp doc man +clean: clean-test clean-dist clean-cache +completions: completion-bash completion-fish completion-zsh doc: README.md CONTRIBUTING.md issuetemplates supportedsites -man: README.txt yt-dlp.1 bash-completion zsh-completion fish-completion +ot: offlinetest +tar: yt-dlp.tar.gz +# Keep this list in sync with MANIFEST.in +# intended use: when building a source distribution, +# make pypi-files && python setup.py sdist +pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites completions yt-dlp.1 devscripts/* test/* -clean: - rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ yt_dlp/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png *.spec *.frag *.frag.urls *.frag.aria2 CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe - find . -name "*.pyc" -delete - find . -name "*.class" -delete +.PHONY: all clean install test tar pypi-files completions ot offlinetest codetest supportedsites + +clean-test: + rm -rf *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png *.frag *.frag.urls *.frag.aria2 +clean-dist: + rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap +clean-cache: + find . -name "*.pyc" -o -name "*.class" -delete + +completion-bash: completions/bash/yt-dlp +completion-fish: completions/fish/yt-dlp.fish +completion-zsh: completions/zsh/_yt-dlp +lazy-extractors: yt_dlp/extractor/lazy_extractors.py PREFIX ?= /usr/local BINDIR ?= $(PREFIX)/bin @@ -21,7 +37,7 @@ SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then ech # set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2 MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi) -install: yt-dlp yt-dlp.1 bash-completion zsh-completion fish-completion +install: yt-dlp yt-dlp.1 completions install -Dm755 yt-dlp $(DESTDIR)$(BINDIR) install -Dm644 yt-dlp.1 $(DESTDIR)$(MANDIR)/man1 install -Dm644 completions/bash/yt-dlp $(DESTDIR)$(SHAREDIR)/bash-completion/completions/yt-dlp @@ -36,8 +52,6 @@ test: nosetests --verbose test $(MAKE) codetest -ot: offlinetest - # Keep this list in sync with devscripts/run_tests.sh offlinetest: codetest $(PYTHON) -m nose --verbose test \ @@ -52,12 +66,6 @@ offlinetest: codetest --exclude test_youtube_signature.py \ --exclude test_post_hooks.py -tar: yt-dlp.tar.gz - -.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites - -pypi-files: README.txt yt-dlp.1 bash-completion zsh-completion fish-completion - yt-dlp: yt_dlp/*.py yt_dlp/*/*.py mkdir -p zip for d in yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor ; do \ @@ -101,28 +109,20 @@ completions/bash/yt-dlp: yt_dlp/*.py yt_dlp/*/*.py devscripts/bash-completion.in mkdir -p completions/bash $(PYTHON) devscripts/bash-completion.py -bash-completion: completions/bash/yt-dlp - completions/zsh/_yt-dlp: yt_dlp/*.py yt_dlp/*/*.py devscripts/zsh-completion.in mkdir -p completions/zsh $(PYTHON) devscripts/zsh-completion.py -zsh-completion: completions/zsh/_yt-dlp - completions/fish/yt-dlp.fish: yt_dlp/*.py yt_dlp/*/*.py devscripts/fish-completion.in mkdir -p completions/fish $(PYTHON) devscripts/fish-completion.py -fish-completion: completions/fish/yt-dlp.fish - -lazy-extractors: yt_dlp/extractor/lazy_extractors.py - _EXTRACTOR_FILES = $(shell find yt_dlp/extractor -iname '*.py' -and -not -iname 'lazy_extractors.py') yt_dlp/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES) $(PYTHON) devscripts/make_lazy_extractors.py $@ -yt-dlp.tar.gz: yt-dlp README.md README.txt yt-dlp.1 bash-completion zsh-completion fish-completion ChangeLog AUTHORS - @tar -czf yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \ +yt-dlp.tar.gz: README.md yt-dlp.1 completions Changelog.md AUTHORS + @tar -czf $(DESTDIR)/yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \ --exclude '*.DS_Store' \ --exclude '*.kate-swp' \ --exclude '*.pyc' \ @@ -132,7 +132,13 @@ yt-dlp.tar.gz: yt-dlp README.md README.txt yt-dlp.1 bash-completion zsh-completi --exclude '.git' \ --exclude 'docs/_build' \ -- \ - bin devscripts test yt_dlp docs \ - ChangeLog AUTHORS LICENSE README.md supportedsites.md README.txt \ + devscripts test \ + Changelog.md AUTHORS LICENSE README.md supportedsites.md \ Makefile MANIFEST.in yt-dlp.1 completions \ setup.py setup.cfg yt-dlp + +AUTHORS: .mailmap + git shortlog -s -n | cut -f2 | sort > AUTHORS + +.mailmap: + git shortlog -s -e -n | awk '!(out[$$NF]++) { $$1="";sub(/^[ \t]+/,""); print}' > .mailmap diff --git a/devscripts/release.sh b/devscripts/release.sh index 86294d56b..d0266f391 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -61,7 +61,7 @@ if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; e if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi -read -p "Is ChangeLog up to date? (y/n) " -n 1 +read -p "Is Changelog up to date? (y/n) " -n 1 if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi /bin/echo -e "\n### First of all, testing..." @@ -75,12 +75,12 @@ fi /bin/echo -e "\n### Changing version in version.py..." sed -i "s/__version__ = '.*'/__version__ = '$version'/" yt_dlp/version.py -/bin/echo -e "\n### Changing version in ChangeLog..." -sed -i "s//$version/" ChangeLog +/bin/echo -e "\n### Changing version in Changelog..." +sed -i "s//$version/" Changelog.md /bin/echo -e "\n### Committing documentation, templates and yt_dlp/version.py..." make README.md CONTRIBUTING.md issuetemplates supportedsites -git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md yt_dlp/version.py ChangeLog +git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md yt_dlp/version.py Changelog.md git commit $gpg_sign_commits -m "release $version" /bin/echo -e "\n### Now tagging, signing and pushing..." @@ -111,7 +111,7 @@ RELEASE_FILES="yt-dlp yt-dlp.exe yt-dlp-$version.tar.gz" for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done ROOT=$(pwd) -python devscripts/create-github-release.py ChangeLog $version "$ROOT/build/$version" +python devscripts/create-github-release.py Changelog.md $version "$ROOT/build/$version" ssh ytdl@yt-dl.org "sh html/update_latest.sh $version" diff --git a/setup.py b/setup.py index 20dbe573c..babdc45af 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,9 @@ if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe': print("inv") else: files_spec = [ - ('share/bash-completion/completions', ['completions/bash/*']), - ('share/zsh/site-functions', ['completions/zsh/*']), - ('share/fish/vendor_completions.d', ['completions/fish/*']), + ('share/bash-completion/completions', ['completions/bash/yt-dlp']), + ('share/zsh/site-functions', ['completions/zsh/_yt-dlp']), + ('share/fish/vendor_completions.d', ['completions/fish/yt-dlp.fish']), ('share/doc/yt_dlp', ['README.txt']), ('share/man/man1', ['yt-dlp.1']) ] @@ -39,7 +39,7 @@ else: resfiles = [] for fn in files: if not os.path.exists(fn): - warnings.warn('Skipping file %s since it is not present. Type make to build all automatically generated files.' % fn) + warnings.warn('Skipping file %s since it is not present. Try running `make pypi-files` first.' % fn) else: resfiles.append(fn) data_files.append((dirname, resfiles))