Backporting Makefile and merge.py.
This commit is contained in:
parent
581fa8ef18
commit
acf16bda91
228
Makefile
228
Makefile
|
@ -2,14 +2,13 @@
|
|||
#
|
||||
# Here is what you can do:
|
||||
#
|
||||
# - make # Automatically build an html local version
|
||||
# - make todo # To list remaining tasks
|
||||
# - make # Automatically build an HTML local version
|
||||
# - make todo # To list remaining tasks and show current progression
|
||||
# - make verifs # To check for correctness: wrapping, spelling
|
||||
# - make wrap # To check for wrapping
|
||||
# - make wrap # To rewrap modified files
|
||||
# - make spell # To check for spelling
|
||||
# - make merge # To merge pot from upstream
|
||||
# - make clean # To remove build artifacts
|
||||
# - make fuzzy # To find fuzzy strings
|
||||
# - make progress # To compute current progression
|
||||
#
|
||||
# Modes are: autobuild-stable, autobuild-dev, and autobuild-html,
|
||||
# documented in gen/src/3.6/Doc/Makefile as we're only delegating the
|
||||
|
@ -22,24 +21,51 @@
|
|||
# test build, we're building with the .rst files that generated our
|
||||
# .po files.
|
||||
CPYTHON_CURRENT_COMMIT := dc3239177ff26cb6a12e437a1f507be730fe8ba7
|
||||
|
||||
CPYTHON_PATH := ../cpython/
|
||||
|
||||
LANGUAGE := fr
|
||||
BRANCH := 3.8
|
||||
BRANCH := 3.9
|
||||
|
||||
.SILENT:
|
||||
EXCLUDED := \
|
||||
whatsnew/2.?.po \
|
||||
whatsnew/3.[0-8].po \
|
||||
c-api/ \
|
||||
distutils/ \
|
||||
install/ \
|
||||
library/2to3.po \
|
||||
library/distutils.po \
|
||||
library/imp.po \
|
||||
library/tkinter.tix.po \
|
||||
library/test.po \
|
||||
library/aifc.po \
|
||||
library/asynchat.po \
|
||||
library/asyncore.po \
|
||||
library/audioop.po \
|
||||
library/cgi.po \
|
||||
library/cgitb.po \
|
||||
library/chunk.po \
|
||||
library/crypt.po \
|
||||
library/imghdr.po \
|
||||
library/msilib.po \
|
||||
library/nntplib.po \
|
||||
library/nis.po \
|
||||
library/ossaudiodev.po \
|
||||
library/pipes.po \
|
||||
library/smtpd.po \
|
||||
library/sndhdr.po \
|
||||
library/spwd.po \
|
||||
library/sunau.po \
|
||||
library/telnetlib.po \
|
||||
library/uu.po \
|
||||
library/xdrlib.po
|
||||
|
||||
# Internal variables
|
||||
|
||||
UPSTREAM := https://github.com/python/cpython
|
||||
VENV := $(shell pwd)/venv/
|
||||
|
||||
PYTHON := $(shell which python3)
|
||||
MODE := html
|
||||
POSPELL_TMP_DIR := .pospell/
|
||||
WORKTREES := $(VENV)/worktrees/
|
||||
WORKTREE := $(WORKTREES)/$(CPYTHON_CURRENT_COMMIT)/
|
||||
JOBS := auto
|
||||
SPHINXERRORHANDLING = -W
|
||||
|
||||
# Detect OS
|
||||
|
||||
|
@ -59,126 +85,104 @@ else
|
|||
endif
|
||||
|
||||
.PHONY: all
|
||||
all: setup
|
||||
mkdir -p $(WORKTREE)/locales/$(LANGUAGE)/LC_MESSAGES/
|
||||
$(CP_CMD) -uv --parents *.po */*.po $(WORKTREE)/locales/$(LANGUAGE)/LC_MESSAGES/ | cut -d"'" -f2
|
||||
$(MAKE) -C $(WORKTREE)/Doc/ VENVDIR=$(WORKTREE)/Doc/venv/ PYTHON=$(PYTHON) \
|
||||
SPHINXOPTS='-qW -j$(JOBS) \
|
||||
-D locale_dirs=../locales \
|
||||
-D language=$(LANGUAGE) \
|
||||
-D gettext_compact=0 \
|
||||
-D latex_engine=xelatex \
|
||||
-D latex_elements.inputenc= \
|
||||
-D latex_elements.fontenc=' \
|
||||
$(MODE) && echo "Build success, open file://$(WORKTREE)/Doc/build/html/index.html or run 'make serve' to see them."
|
||||
all: ensure_prerequisites
|
||||
git -C venv/cpython checkout $(CPYTHON_CURRENT_COMMIT) || (git -C venv/cpython fetch && git -C venv/cpython checkout $(CPYTHON_CURRENT_COMMIT))
|
||||
mkdir -p locales/$(LANGUAGE)/LC_MESSAGES/
|
||||
$(CP_CMD) -u --parents *.po */*.po locales/$(LANGUAGE)/LC_MESSAGES/
|
||||
$(MAKE) -C venv/cpython/Doc/ \
|
||||
JOBS='$(JOBS)' \
|
||||
SPHINXOPTS='-D locale_dirs=$(abspath locales) \
|
||||
-D language=$(LANGUAGE) \
|
||||
-D gettext_compact=0 \
|
||||
-D latex_engine=xelatex \
|
||||
-D latex_elements.inputenc= \
|
||||
-D latex_elements.fontenc=' \
|
||||
SPHINXERRORHANDLING=$(SPHINXERRORHANDLING) \
|
||||
$(MODE)
|
||||
@echo "Build success, open file://$(abspath venv/cpython/)/Doc/build/html/index.html or run 'make htmlview' to see them."
|
||||
|
||||
|
||||
.PHONY: setup
|
||||
setup: venv
|
||||
# Setup the main clone
|
||||
if ! [ -d $(CPYTHON_PATH) ]; then \
|
||||
git clone --depth 16 --branch $(BRANCH) $(UPSTREAM) $(CPYTHON_PATH); \
|
||||
# We clone cpython/ inside venv/ because venv/ is the only directory
|
||||
# excluded by cpython' Sphinx configuration.
|
||||
venv/cpython/.git/HEAD:
|
||||
git clone https://github.com/python/cpython venv/cpython
|
||||
|
||||
|
||||
.PHONY: ensure_prerequisites
|
||||
ensure_prerequisites: venv/cpython/.git/HEAD
|
||||
@if ! (blurb help >/dev/null 2>&1 && sphinx-build --version >/dev/null 2>&1); then \
|
||||
git -C venv/cpython/ checkout $(BRANCH); \
|
||||
echo "You're missing dependencies please install:"; \
|
||||
echo ""; \
|
||||
echo " python -m pip install -r requirements.txt -r venv/cpython/Doc/requirements.txt"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Setup the current worktree
|
||||
if ! [ -d $(WORKTREE) ]; then \
|
||||
rm -fr $(WORKTREES); \
|
||||
git -C $(CPYTHON_PATH) worktree prune; \
|
||||
mkdir -p $(WORKTREES); \
|
||||
if [ -n "$(CPYTHON_CURRENT_COMMIT)" ]; \
|
||||
then \
|
||||
depth=32; \
|
||||
while ! git -C $(CPYTHON_PATH) cat-file -e $(CPYTHON_CURRENT_COMMIT); \
|
||||
do \
|
||||
depth=$$((depth * 2)); \
|
||||
git -C $(CPYTHON_PATH) fetch --depth $$depth $(UPSTREAM) $(BRANCH); \
|
||||
done \
|
||||
else \
|
||||
git -C $(CPYTHON_PATH) fetch --depth 1 $(UPSTREAM); \
|
||||
fi; \
|
||||
git -C $(CPYTHON_PATH) worktree add $(WORKTREE)/ $(CPYTHON_CURRENT_COMMIT); \
|
||||
$(MAKE) -C $(WORKTREE)/Doc/ VENVDIR=$(WORKTREE)/Doc/venv/ PYTHON=$(PYTHON) venv; \
|
||||
$(WORKTREE)/Doc/venv/bin/python -m pip install Sphinx==2.2.2 docutils==0.15; \
|
||||
fi
|
||||
|
||||
|
||||
.PHONY: venv
|
||||
venv:
|
||||
if [ ! -d $(VENV) ]; then $(PYTHON) -m venv --prompt python-docs-fr $(VENV); fi
|
||||
$(VENV)/bin/python -m pip install -q -r requirements.txt 2> $(VENV)/pip-install.log
|
||||
if grep -q 'pip install --upgrade pip' $(VENV)/pip-install.log; then \
|
||||
$(VENV)/bin/pip install -q --upgrade pip; \
|
||||
fi
|
||||
|
||||
|
||||
.PHONY: serve
|
||||
serve:
|
||||
$(MAKE) -C $(WORKTREE)/Doc/ serve
|
||||
|
||||
|
||||
.PHONY: progress
|
||||
progress:
|
||||
$(PYTHON) -c 'import sys; print("{:.1%}".format(int(sys.argv[1]) / int(sys.argv[2])))' \
|
||||
$(shell msgcat *.po */*.po | msgattrib --translated | grep -c '^msgid') \
|
||||
$(shell msgcat *.po */*.po | grep -c '^msgid')
|
||||
|
||||
.PHONY: htmlview
|
||||
htmlview: MODE=htmlview
|
||||
htmlview: all
|
||||
|
||||
.PHONY: todo
|
||||
todo: venv
|
||||
$(VENV)/bin/potodo
|
||||
todo: ensure_prerequisites
|
||||
potodo --api-url 'https://git.afpy.org/api/v1/repos/AFPy/python-docs-fr/issues?state=open&type=issues' --exclude venv .venv $(EXCLUDED)
|
||||
|
||||
.PHONY: wrap
|
||||
wrap: venv
|
||||
$(VENV)/bin/powrap --check --quiet *.po **/*.po
|
||||
wrap: ensure_prerequisites
|
||||
@echo "Re wrapping modified files"
|
||||
powrap -m
|
||||
|
||||
SRCS = $(shell git diff --name-only $(BRANCH) | grep '.po$$')
|
||||
# foo/bar.po => $(POSPELL_TMP_DIR)/foo/bar.po.out
|
||||
DESTS = $(addprefix $(POSPELL_TMP_DIR)/,$(addsuffix .out,$(SRCS)))
|
||||
|
||||
.PHONY: spell
|
||||
spell: venv $(DESTS)
|
||||
spell: ensure_prerequisites $(DESTS)
|
||||
|
||||
.PHONY: line-length
|
||||
line-length:
|
||||
@echo "Searching for long lines..."
|
||||
@awk '{if (length(gensub(/శ్రీనివాస్/, ".", "g", $$0)) > 80 && length(gensub(/[^ ]/, "", "g")) > 1) {print FILENAME ":" FNR, "line too long:", $$0; ERRORS+=1}} END {if (ERRORS>0) {exit 1}}' *.po */*.po
|
||||
|
||||
.PHONY: sphinx-lint
|
||||
sphinx-lint:
|
||||
@echo "Checking all files using sphinx-lint..."
|
||||
@sphinx-lint --enable all --disable line-too-long *.po */*.po
|
||||
|
||||
$(POSPELL_TMP_DIR)/%.po.out: %.po dict
|
||||
echo "Checking $<..."
|
||||
mkdir -p $(@D)
|
||||
$(VENV)/bin/pospell -p dict -l fr_FR $< && touch $@
|
||||
@echo "Pospell checking $<..."
|
||||
@mkdir -p $(@D)
|
||||
pospell -p dict -l fr_FR $< && touch $@
|
||||
|
||||
.PHONY: fuzzy
|
||||
fuzzy: venv
|
||||
$(VENV)/bin/potodo -f
|
||||
fuzzy: ensure_prerequisites
|
||||
potodo --only-fuzzy --api-url 'https://git.afpy.org/api/v1/repos/AFPy/python-docs-fr/issues?state=open&type=issues' --exclude venv .venv $(EXCLUDED)
|
||||
|
||||
.PHONY: check-headers
|
||||
check-headers:
|
||||
@grep -L '^# Copyright (C) [0-9-]*, Python Software Foundation' *.po */*.po | while read -r file;\
|
||||
do \
|
||||
echo "Please update the po comment in $$file"; \
|
||||
done
|
||||
@grep -L '^"Project-Id-Version: Python 3\\n"$$' *.po */*.po | while read -r file;\
|
||||
do \
|
||||
echo "Please update the 'Project-Id-Version' header in $$file"; \
|
||||
done
|
||||
@grep -L '^"Language: fr\\n"$$' *.po */*.po | while read -r file;\
|
||||
do \
|
||||
echo "Please update the 'Language' header in $$file"; \
|
||||
done
|
||||
@grep -L '^"Language-Team: FRENCH <traductions@lists.afpy.org>\\n"' *.po */*.po | while read -r file;\
|
||||
do \
|
||||
echo "Please update the 'Language-Team' header in $$file"; \
|
||||
done
|
||||
|
||||
.PHONY: verifs
|
||||
verifs: wrap spell
|
||||
|
||||
.PHONY: merge
|
||||
merge: setup
|
||||
git -C $(CPYTHON_PATH) fetch $(UPSTREAM)
|
||||
rm -fr $(WORKTREES)/$(BRANCH)
|
||||
git -C $(CPYTHON_PATH) worktree prune
|
||||
git -C $(CPYTHON_PATH) worktree add $(WORKTREES)/$(BRANCH) $(word 1,$(shell git -C $(CPYTHON_PATH) remote -v | grep python/cpython))/$(BRANCH)
|
||||
$(MAKE) -C $(WORKTREES)/$(BRANCH)/Doc/ VENVDIR=$(WORKTREES)/$(BRANCH)/Doc/venv/ PYTHON=$(PYTHON) venv;
|
||||
(cd $(WORKTREES)/$(BRANCH); $(WORKTREES)/$(BRANCH)/Doc/venv/bin/sphinx-build -Q -b gettext -D gettext_compact=0 Doc pot/)
|
||||
find $(WORKTREES)/$(BRANCH) -name '*.pot' |\
|
||||
while read -r POT;\
|
||||
do\
|
||||
PO="./$$(echo "$$POT" | sed "s#$(WORKTREES)/$(BRANCH)/pot/##; s#\.pot\$$#.po#")";\
|
||||
mkdir -p "$$(dirname "$$PO")";\
|
||||
if [ -f "$$PO" ];\
|
||||
then\
|
||||
case "$$POT" in\
|
||||
*whatsnew*) msgmerge --backup=off --force-po --no-fuzzy-matching -U "$$PO" "$$POT" ;;\
|
||||
*) msgmerge --backup=off --force-po -U "$$PO" "$$POT" ;;\
|
||||
esac\
|
||||
else\
|
||||
msgcat -o "$$PO" "$$POT";\
|
||||
fi\
|
||||
done
|
||||
sed -i 's/^CPYTHON_CURRENT_COMMIT :=.*/CPYTHON_CURRENT_COMMIT := $(shell git -C $(WORKTREES)/$(BRANCH) rev-parse HEAD)/' Makefile
|
||||
rm -fr $(WORKTREES)/$(BRANCH)
|
||||
git -C $(CPYTHON_PATH) worktree prune
|
||||
echo 'To add, you can use git status -s | grep "^ M .*\.po" | cut -d" " -f3 | while read -r file; do if [ $$(git diff "$$file" | wc -l) -gt 13 ]; then git add "$$file"; fi ; done'
|
||||
verifs: spell line-length sphinx-lint check-headers
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -fr $(VENV) $(POSPELL_TMP_DIR)
|
||||
@echo "Cleaning *.mo and $(POSPELL_TMP_DIR)"
|
||||
rm -fr $(POSPELL_TMP_DIR) locales/$(LANGUAGE)/LC_MESSAGES/
|
||||
find -name '*.mo' -delete
|
||||
@echo "Cleaning build directory"
|
||||
$(MAKE) -C venv/cpython/Doc/ clean
|
||||
|
|
150
merge.py
Normal file
150
merge.py
Normal file
|
@ -0,0 +1,150 @@
|
|||
"""Tool to merge cpython pot files to python-docs-fr po files for a
|
||||
given branch.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
NOT_TO_TRANSLATE = {Path("whatsnew/changelog.po")}
|
||||
|
||||
|
||||
def run(*args: str | Path, **kwargs) -> subprocess.CompletedProcess:
|
||||
"""Run a shell command with subprocess.run() with check=True and
|
||||
encoding="UTF-8".
|
||||
"""
|
||||
return subprocess.run(list(args), encoding="UTF-8", check=True, **kwargs)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--cpython_repo",
|
||||
default=Path("venv/cpython"),
|
||||
type=Path,
|
||||
help="Use this given cpython clone.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"branch",
|
||||
help="Merge from this branch or from this commit",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def setup_repo(repo_path: Path, branch: str):
|
||||
"""Ensure we're up-to-date."""
|
||||
if branch.find('.') == 1:
|
||||
run("git", "-C", repo_path, "checkout", branch)
|
||||
run("git", "-C", repo_path, "pull", "--ff-only")
|
||||
else: # it's a commit
|
||||
run("git", "-C", repo_path, "checkout", branch)
|
||||
|
||||
|
||||
def copy_new_files(new_files: set[Path], pot_path: Path) -> None:
|
||||
"""Just copy new po files to our hierarchy."""
|
||||
print(f"{len(new_files)} new files.")
|
||||
for file in new_files:
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
src = (pot_path / file).with_suffix(".pot")
|
||||
run("msgcat", "-o", file, src)
|
||||
run("git", "add", file)
|
||||
|
||||
|
||||
def update_known_files(known_files: set[Path], pot_path: Path) -> None:
|
||||
"""msgmerge updated pot files in our po files."""
|
||||
print(f"{len(known_files)} files to update.")
|
||||
for file in tqdm(known_files, desc="merging pot files"):
|
||||
src = (pot_path / file).with_suffix(".pot")
|
||||
run("msgmerge", "-q", "--backup=off", "--force-po", "-U", file, src)
|
||||
|
||||
|
||||
def remove_old_files(old_files: set[Path]) -> None:
|
||||
"""Remove files removed upstream."""
|
||||
print(f"{len(old_files)} removed files.")
|
||||
|
||||
for file in old_files:
|
||||
run("git", "rm", file)
|
||||
|
||||
|
||||
def clean_paths(files: set[Path]) -> None:
|
||||
"""Ensure the path present in po files are always relative.
|
||||
|
||||
This avoid having diffs on those paths when we change something in
|
||||
a script.
|
||||
"""
|
||||
for file in tqdm(files, desc="Cleaning rst path in pot files"):
|
||||
contents = file.read_text(encoding="UTF-8")
|
||||
contents = re.sub("^#: .*Doc/", "#: ", contents, flags=re.M)
|
||||
file.write_text(contents, encoding="UTF-8")
|
||||
|
||||
|
||||
def update_makefile(cpython_repo: Path) -> None:
|
||||
"""Update CPYTHON_CURRENT_COMMIT in the Makefile.
|
||||
|
||||
So that when we run `make` it use the same commit than the one
|
||||
used to generate the `po` files.
|
||||
"""
|
||||
makefile = Path("Makefile").read_text(encoding="UTF-8")
|
||||
head = run(
|
||||
"git", "-C", cpython_repo, "rev-parse", "HEAD", stdout=PIPE
|
||||
).stdout.strip()
|
||||
makefile = re.sub(
|
||||
"^CPYTHON_CURRENT_COMMIT :=.*$",
|
||||
f"CPYTHON_CURRENT_COMMIT := {head}",
|
||||
makefile,
|
||||
flags=re.M,
|
||||
)
|
||||
Path("Makefile").write_text(makefile, encoding="UTF-8")
|
||||
run("git", "add", "Makefile")
|
||||
|
||||
|
||||
def git_add_relevant_files():
|
||||
"""Add only files with relevant modifications.
|
||||
|
||||
This only add files with actual modifications, not just metadata
|
||||
modifications, to avoid noise in history.
|
||||
"""
|
||||
modified_files = run("git", "ls-files", "-m", stdout=PIPE).stdout.split("\n")
|
||||
modified_po_files = [line for line in modified_files if line.endswith(".po")]
|
||||
for file in modified_po_files:
|
||||
diff = run("git", "diff", "-U0", file, stdout=PIPE).stdout
|
||||
if len(diff.split("\n")) > 8:
|
||||
run("git", "add", file)
|
||||
else:
|
||||
run("git", "checkout", "--", file)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
setup_repo(args.cpython_repo, args.branch)
|
||||
run(
|
||||
*["sphinx-build", "-jauto", "-QDgettext_compact=0", "-bgettext", ".", "../pot"],
|
||||
cwd=args.cpython_repo / "Doc",
|
||||
)
|
||||
pot_path = args.cpython_repo / "pot"
|
||||
upstream = {
|
||||
file.relative_to(pot_path).with_suffix(".po")
|
||||
for file in pot_path.glob("**/*.pot")
|
||||
} - NOT_TO_TRANSLATE
|
||||
downstream = {
|
||||
Path(po)
|
||||
for po in run("git", "ls-files", "*.po", stdout=PIPE).stdout.splitlines()
|
||||
}
|
||||
copy_new_files(upstream - downstream, pot_path=pot_path)
|
||||
update_known_files(upstream & downstream, pot_path=pot_path)
|
||||
remove_old_files(downstream - upstream)
|
||||
clean_paths(upstream)
|
||||
shutil.rmtree(pot_path)
|
||||
run("powrap", "-m")
|
||||
update_makefile(args.cpython_repo)
|
||||
git_add_relevant_files()
|
||||
run("git", "commit", "-m", "Make merge")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Loading…
Reference in New Issue
Block a user