From e7b27fcd03ecd83fe8fa6d19bf08c98ac3290678 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 22 Mar 2023 22:46:57 +0100 Subject: [PATCH] Backport MAkefile and merge.py --- Makefile | 172 ++++++++++++++++++++++++++++++------------------------- merge.py | 150 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+), 78 deletions(-) create mode 100644 merge.py diff --git a/Makefile b/Makefile index 57bc23d7..f598de64 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,12 @@ # Here is what you can do: # # - make # Automatically build an HTML local version -# - make todo # To list remaining tasks +# - make todo # To list remaining tasks and show current progression # - make verifs # To check for correctness: wrapping, spelling -# - make wrap # To check for wrapping +# - make wrap # To rewrap modified files # - make spell # To check for spelling -# - make merge # To merge pot from upstream +# - make clean # To remove build artifacts # - make fuzzy # To find fuzzy strings -# - make progress # To compute current progression # # Modes are: autobuild-stable, autobuild-dev, and autobuild-html, # documented in gen/src/3.6/Doc/Makefile as we're only delegating the @@ -22,13 +21,41 @@ # test build, we're building with the .rst files that generated our # .po files. CPYTHON_CURRENT_COMMIT := cb0b009cfb4ae2a87a8737250c526b8f95d0551f - -CPYTHON_PATH := ../cpython/ - LANGUAGE := fr BRANCH := 3.9 -EXCLUDED := whatsnew/ c-api/ +EXCLUDED := \ + whatsnew/2.?.po \ + whatsnew/3.[0-8].po \ + c-api/ \ + distutils/ \ + install/ \ + library/2to3.po \ + library/distutils.po \ + library/imp.po \ + library/tkinter.tix.po \ + library/test.po \ + library/aifc.po \ + library/asynchat.po \ + library/asyncore.po \ + library/audioop.po \ + library/cgi.po \ + library/cgitb.po \ + library/chunk.po \ + library/crypt.po \ + library/imghdr.po \ + library/msilib.po \ + library/nntplib.po \ + library/nis.po \ + library/ossaudiodev.po \ + library/pipes.po \ + library/smtpd.po \ + library/sndhdr.po \ + library/spwd.po \ + library/sunau.po \ + library/telnetlib.po \ + library/uu.po \ + library/xdrlib.po # Internal variables @@ -38,6 +65,7 @@ PYTHON := $(shell which python3) MODE := html POSPELL_TMP_DIR := .pospell/ JOBS := auto +SPHINXERRORHANDLING = -W # Detect OS @@ -58,69 +86,50 @@ endif .PHONY: all all: ensure_prerequisites - git -C $(CPYTHON_PATH) checkout $(CPYTHON_CURRENT_COMMIT) + git -C venv/cpython checkout $(CPYTHON_CURRENT_COMMIT) || (git -C venv/cpython fetch && git -C venv/cpython checkout $(CPYTHON_CURRENT_COMMIT)) mkdir -p locales/$(LANGUAGE)/LC_MESSAGES/ $(CP_CMD) -u --parents *.po */*.po locales/$(LANGUAGE)/LC_MESSAGES/ - $(MAKE) -C $(CPYTHON_PATH)/Doc/ \ - SPHINXOPTS='-W -j$(JOBS) \ - -D locale_dirs=$(abspath locales) \ + $(MAKE) -C venv/cpython/Doc/ \ + JOBS='$(JOBS)' \ + SPHINXOPTS='-D locale_dirs=$(abspath locales) \ -D language=$(LANGUAGE) \ - -D gettext_compact=0 \ + -D gettext_compact=0 \ -D latex_engine=xelatex \ -D latex_elements.inputenc= \ -D latex_elements.fontenc=' \ + SPHINXERRORHANDLING=$(SPHINXERRORHANDLING) \ $(MODE) - @echo "Build success, open file://$(abspath $(CPYTHON_PATH))/Doc/build/html/index.html or run 'make serve' to see them." + @echo "Build success, open file://$(abspath venv/cpython/)/Doc/build/html/index.html or run 'make htmlview' to see them." + + +# We clone cpython/ inside venv/ because venv/ is the only directory +# excluded by cpython' Sphinx configuration. +venv/cpython/.git/HEAD: + git clone https://github.com/python/cpython venv/cpython .PHONY: ensure_prerequisites -ensure_prerequisites: - @if [ -z $(CPYTHON_PATH) ]; then \ - echo "Your CPYTHON_PATH is empty, please provide one."; \ - exit 1; \ - fi - @if ! [ -d $(CPYTHON_PATH) ]; then \ - echo "Building the translation requires a cpython clone."; \ - echo "Please provide the path to a clone using the CPYTHON_PATH variable."; \ - echo "(Currently CPYTHON_PATH is $(CPYTHON_PATH)."; \ - echo "So you may want to run:"; \ - echo ""; \ - echo " git clone $(UPSTREAM) $(CPYTHON_PATH)"; \ - exit 1; \ - fi - @if [ -n "$$(git -C $(CPYTHON_PATH) status --porcelain)" ]; then \ - echo "Your cpython clone at $(CPYTHON_PATH) is not clean."; \ - echo "In order to avoid breaking things, please clean it first."; \ - exit 1; \ - fi +ensure_prerequisites: venv/cpython/.git/HEAD @if ! (blurb help >/dev/null 2>&1 && sphinx-build --version >/dev/null 2>&1); then \ - git -C $(CPYTHON_PATH) checkout $(BRANCH); \ - echo "You're missing dependencies, please enable a venv and install:"; \ + git -C venv/cpython/ checkout $(BRANCH); \ + echo "You're missing dependencies please install:"; \ echo ""; \ - echo " python -m pip install -r requirements.txt -r $(CPYTHON_PATH)/Doc/requirements.txt"; \ + echo " python -m pip install -r requirements.txt -r venv/cpython/Doc/requirements.txt"; \ exit 1; \ fi -.PHONY: serve -serve: - $(MAKE) -C $(CPYTHON_PATH)/Doc/ serve - - -.PHONY: progress -progress: - @$(PYTHON) -c 'import sys; print("{:.1%}".format(int(sys.argv[1]) / int(sys.argv[2])))' \ - $(shell msgcat *.po */*.po | msgattrib --translated | grep -c '^msgid') \ - $(shell msgcat *.po */*.po | grep -c '^msgid') - +.PHONY: htmlview +htmlview: MODE=htmlview +htmlview: all .PHONY: todo todo: ensure_prerequisites - potodo --exclude venv .venv $(EXCLUDED) + potodo --api-url 'https://git.afpy.org/api/v1/repos/AFPy/python-docs-fr/issues?state=open&type=issues' --exclude venv .venv $(EXCLUDED) .PHONY: wrap wrap: ensure_prerequisites - @echo "Verify wrapping" - powrap --check --quiet *.po **/*.po + @echo "Re wrapping modified files" + powrap -m SRCS = $(shell git diff --name-only $(BRANCH) | grep '.po$$') # foo/bar.po => $(POSPELL_TMP_DIR)/foo/bar.po.out @@ -129,44 +138,51 @@ DESTS = $(addprefix $(POSPELL_TMP_DIR)/,$(addsuffix .out,$(SRCS))) .PHONY: spell spell: ensure_prerequisites $(DESTS) +.PHONY: line-length +line-length: + @echo "Searching for long lines..." + @awk '{if (length(gensub(/శ్రీనివాస్/, ".", "g", $$0)) > 80 && length(gensub(/[^ ]/, "", "g")) > 1) {print FILENAME ":" FNR, "line too long:", $$0; ERRORS+=1}} END {if (ERRORS>0) {exit 1}}' *.po */*.po + +.PHONY: sphinx-lint +sphinx-lint: + @echo "Checking all files using sphinx-lint..." + @sphinx-lint --enable all --disable line-too-long *.po */*.po + $(POSPELL_TMP_DIR)/%.po.out: %.po dict @echo "Pospell checking $<..." - mkdir -p $(@D) + @mkdir -p $(@D) pospell -p dict -l fr_FR $< && touch $@ .PHONY: fuzzy fuzzy: ensure_prerequisites - potodo -f --exclude venv .venv $(EXCLUDED) + potodo --only-fuzzy --api-url 'https://git.afpy.org/api/v1/repos/AFPy/python-docs-fr/issues?state=open&type=issues' --exclude venv .venv $(EXCLUDED) + +.PHONY: check-headers +check-headers: + @grep -L '^# Copyright (C) [0-9-]*, Python Software Foundation' *.po */*.po | while read -r file;\ + do \ + echo "Please update the po comment in $$file"; \ + done + @grep -L '^"Project-Id-Version: Python 3\\n"$$' *.po */*.po | while read -r file;\ + do \ + echo "Please update the 'Project-Id-Version' header in $$file"; \ + done + @grep -L '^"Language: fr\\n"$$' *.po */*.po | while read -r file;\ + do \ + echo "Please update the 'Language' header in $$file"; \ + done + @grep -L '^"Language-Team: FRENCH \\n"' *.po */*.po | while read -r file;\ + do \ + echo "Please update the 'Language-Team' header in $$file"; \ + done .PHONY: verifs -verifs: wrap spell - -.PHONY: merge -merge: ensure_prerequisites - @echo "Merge from $(UPSTREAM)" - git -C $(CPYTHON_PATH) checkout $(BRANCH) - git -C $(CPYTHON_PATH) pull --ff-only - (cd $(CPYTHON_PATH)/Doc; sphinx-build -Q -b gettext -D gettext_compact=0 . ../pot) - find $(CPYTHON_PATH)/pot/ -name '*.pot' |\ - while read -r POT; \ - do \ - PO="./$$(echo "$$POT" | sed "s#$(CPYTHON_PATH)/pot/##; s#\.pot\$$#.po#")"; \ - mkdir -p "$$(dirname "$$PO")"; \ - if [ -f "$$PO" ]; \ - then \ - msgmerge --backup=off --force-po -U "$$PO" "$$POT"; \ - else \ - msgcat -o "$$PO" "$$POT"; \ - fi \ - done - rm -fr $(CPYTHON_PATH)/pot/ - sed -i 's|^#: .*Doc/|#: |' *.po */*.po - powrap -m - @printf "\n%s %s\n" "Replace CPYTHON_CURRENT_COMMIT in Makefile by: " $(shell git -C $(CPYTHON_PATH) rev-parse HEAD) - @printf 'To add, you can use:\n git status -s | grep "^ M .*\.po" | cut -d" " -f3 | while read -r file; do if [ $$(git diff "$$file" | wc -l) -gt 13 ]; then git add "$$file"; fi ; done\n' +verifs: spell line-length sphinx-lint check-headers .PHONY: clean clean: @echo "Cleaning *.mo and $(POSPELL_TMP_DIR)" - rm -fr $(POSPELL_TMP_DIR) + rm -fr $(POSPELL_TMP_DIR) locales/$(LANGUAGE)/LC_MESSAGES/ find -name '*.mo' -delete + @echo "Cleaning build directory" + $(MAKE) -C venv/cpython/Doc/ clean diff --git a/merge.py b/merge.py new file mode 100644 index 00000000..6cc9f608 --- /dev/null +++ b/merge.py @@ -0,0 +1,150 @@ +"""Tool to merge cpython pot files to python-docs-fr po files for a +given branch. +""" + +import argparse +import re +import shutil +import subprocess +from pathlib import Path +from subprocess import PIPE + +from tqdm import tqdm + +NOT_TO_TRANSLATE = {Path("whatsnew/changelog.po")} + + +def run(*args: str | Path, **kwargs) -> subprocess.CompletedProcess: + """Run a shell command with subprocess.run() with check=True and + encoding="UTF-8". + """ + return subprocess.run(list(args), encoding="UTF-8", check=True, **kwargs) + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cpython_repo", + default=Path("venv/cpython"), + type=Path, + help="Use this given cpython clone.", + ) + parser.add_argument( + "branch", + help="Merge from this branch or from this commit", + ) + return parser.parse_args() + + +def setup_repo(repo_path: Path, branch: str): + """Ensure we're up-to-date.""" + if branch.find('.') == 1: + run("git", "-C", repo_path, "checkout", branch) + run("git", "-C", repo_path, "pull", "--ff-only") + else: # it's a commit + run("git", "-C", repo_path, "checkout", branch) + + +def copy_new_files(new_files: set[Path], pot_path: Path) -> None: + """Just copy new po files to our hierarchy.""" + print(f"{len(new_files)} new files.") + for file in new_files: + file.parent.mkdir(parents=True, exist_ok=True) + src = (pot_path / file).with_suffix(".pot") + run("msgcat", "-o", file, src) + run("git", "add", file) + + +def update_known_files(known_files: set[Path], pot_path: Path) -> None: + """msgmerge updated pot files in our po files.""" + print(f"{len(known_files)} files to update.") + for file in tqdm(known_files, desc="merging pot files"): + src = (pot_path / file).with_suffix(".pot") + run("msgmerge", "-q", "--backup=off", "--force-po", "-U", file, src) + + +def remove_old_files(old_files: set[Path]) -> None: + """Remove files removed upstream.""" + print(f"{len(old_files)} removed files.") + + for file in old_files: + run("git", "rm", file) + + +def clean_paths(files: set[Path]) -> None: + """Ensure the path present in po files are always relative. + + This avoid having diffs on those paths when we change something in + a script. + """ + for file in tqdm(files, desc="Cleaning rst path in pot files"): + contents = file.read_text(encoding="UTF-8") + contents = re.sub("^#: .*Doc/", "#: ", contents, flags=re.M) + file.write_text(contents, encoding="UTF-8") + + +def update_makefile(cpython_repo: Path) -> None: + """Update CPYTHON_CURRENT_COMMIT in the Makefile. + + So that when we run `make` it use the same commit than the one + used to generate the `po` files. + """ + makefile = Path("Makefile").read_text(encoding="UTF-8") + head = run( + "git", "-C", cpython_repo, "rev-parse", "HEAD", stdout=PIPE + ).stdout.strip() + makefile = re.sub( + "^CPYTHON_CURRENT_COMMIT :=.*$", + f"CPYTHON_CURRENT_COMMIT := {head}", + makefile, + flags=re.M, + ) + Path("Makefile").write_text(makefile, encoding="UTF-8") + run("git", "add", "Makefile") + + +def git_add_relevant_files(): + """Add only files with relevant modifications. + + This only add files with actual modifications, not just metadata + modifications, to avoid noise in history. + """ + modified_files = run("git", "ls-files", "-m", stdout=PIPE).stdout.split("\n") + modified_po_files = [line for line in modified_files if line.endswith(".po")] + for file in modified_po_files: + diff = run("git", "diff", "-U0", file, stdout=PIPE).stdout + if len(diff.split("\n")) > 8: + run("git", "add", file) + else: + run("git", "checkout", "--", file) + + +def main(): + args = parse_args() + setup_repo(args.cpython_repo, args.branch) + run( + *["sphinx-build", "-jauto", "-QDgettext_compact=0", "-bgettext", ".", "../pot"], + cwd=args.cpython_repo / "Doc", + ) + pot_path = args.cpython_repo / "pot" + upstream = { + file.relative_to(pot_path).with_suffix(".po") + for file in pot_path.glob("**/*.pot") + } - NOT_TO_TRANSLATE + downstream = { + Path(po) + for po in run("git", "ls-files", "*.po", stdout=PIPE).stdout.splitlines() + } + copy_new_files(upstream - downstream, pot_path=pot_path) + update_known_files(upstream & downstream, pot_path=pot_path) + remove_old_files(downstream - upstream) + clean_paths(upstream) + shutil.rmtree(pot_path) + run("powrap", "-m") + update_makefile(args.cpython_repo) + git_add_relevant_files() + run("git", "commit", "-m", "Make merge") + + +if __name__ == "__main__": + main()