begin refactoring

This commit is contained in:
Vincent Poulailleau 2019-11-22 12:50:24 +01:00
parent 9439bd15bc
commit a31af33018
11 changed files with 467 additions and 368 deletions

View File

View File

@ -0,0 +1,35 @@
"""Base class for checkers."""
from abc import ABC, abstractmethod
import simplelogging
from padpo.pofile import PoFile, PoItem
log = simplelogging.get_logger()
class Checker(ABC):
"""Base class for checkers."""
def __init__(self, name):
"""Initializer, name is displayed in messages."""
self.name = name
def check_file(self, pofile: PoFile):
"""Check a `*.po` file."""
if not isinstance(pofile, PoFile):
log.error("%s is not an instance of PoFile", str(pofile))
for item in pofile.content:
self.check_item(item)
@abstractmethod
def check_item(self, item: PoItem):
"""Check an item in a `*.po` file."""
return NotImplementedError
def replace_quotes(match):
"""Replace match with « xxxxxxx »."""
length = len(match.group(0)) - 4
return "« " + length * "x" + " »"

View File

@ -0,0 +1,25 @@
"""Checker for double spaces."""
import re
from padpo.checkers.baseclass import Checker
from padpo.pofile import PoItem
class DoubleSpaceChecker(Checker):
"""Checker for double spaces."""
def __init__(self):
"""Initialiser."""
super().__init__(name="Double space")
def check_item(self, item: PoItem):
"""Check an item in a `*.po` file."""
for match in re.finditer(
r"(.{0,30})\s\s(.{0,30})", item.msgstr_full_content
):
item.add_warning(
self.name,
f"Double spaces detected between ###{match.group(1)}### "
f"and ###{match.group(2)}###",
)

18
padpo/checkers/empty.py Normal file
View File

@ -0,0 +1,18 @@
"""Checker for missing translations."""
from padpo.checkers.baseclass import Checker
from padpo.pofile import PoItem
class EmptyChecker(Checker):
"""Checker for missing translations."""
def __init__(self):
"""Initialiser."""
super().__init__(name="Empty")
def check_item(self, item: PoItem):
"""Check an item in a `*.po` file."""
if item.msgid_full_content and not item.msgstr_full_content:
item.add_warning(self.name, "This entry is not translated yet.")

17
padpo/checkers/fuzzy.py Normal file
View File

@ -0,0 +1,17 @@
"""Checker for fuzzy translations."""
from padpo.checkers.baseclass import Checker
from padpo.pofile import PoItem
class FuzzyChecker(Checker):
"""Checker for fuzzy translations."""
def __init__(self):
"""Initialiser."""
super().__init__(name="Fuzzy")
def check_item(self, item: PoItem):
"""Check an item in a `*.po` file."""
if item.fuzzy:
item.add_warning(self.name, "This entry is tagged as fuzzy.")

View File

@ -0,0 +1,119 @@
"""Checker for grammar errors."""
import json
import re
import subprocess
import tempfile
from pathlib import Path
from zipfile import ZipFile
import requests
import simplelogging
from padpo.checkers.baseclass import Checker, replace_quotes
from padpo.pofile import PoItem, PoFile
log = simplelogging.get_logger()
class GrammalecteChecker(Checker):
"""Checker for grammar errors."""
def __init__(self):
"""Initialiser."""
super().__init__(name="Grammalecte")
self.dir = None
def check_file(self, pofile: PoFile):
"""Check a `*.po` file."""
if not isinstance(pofile, PoFile):
log.error("%s is not an instance of PoFile", str(pofile))
fd, name = tempfile.mkstemp(suffix=".txt", prefix="padpo_", text=True)
with open(name, "w", encoding="utf8") as f:
text = pofile.rst2txt()
text = re.sub(r"«\s(.*?)\", replace_quotes, text)
f.write(text)
try:
result = subprocess.run(
[
"grammalecte-cli.py",
"-f",
name,
"-off",
"apos",
"--json",
"--only_when_errors",
],
capture_output=True,
text=True,
)
except FileNotFoundError as e:
if e.filename == "grammalecte-cli.py":
install_grammalecte()
result = subprocess.run(
[
"grammalecte-cli.py",
"-f",
name,
"-off",
"apos",
"--json",
"--only_when_errors",
],
capture_output=True,
text=True,
)
if result.stdout:
warnings = json.loads(result.stdout)
for warning in warnings["data"]:
for error in warning["lGrammarErrors"]:
if self.filter_out(error):
continue
item_index = int(warning["iParagraph"]) // 2
item = pofile.content[item_index]
start = max(0, int(error["nStart"]) - 40)
end = max(0, int(error["nEnd"]) + 10)
item.add_warning(
self.name,
# self.name + " " + error["sRuleId"], # TODO
error["sMessage"]
+ " => ###"
+ item.msgstr_rst2txt[start:end]
+ "###",
)
Path(name).unlink()
def check_item(self, item: PoItem):
"""Check an item in a `*.po` file (does nothing)."""
pass
def filter_out(self, error):
"""Return True when grammalecte error should be ignored."""
msg = error["sRuleId"]
if msg == "esp_milieu_ligne":
return True # double space
if msg == "nbsp_avant_deux_points":
return True
if msg == "nbsp_avant_double_ponctuation":
return True
return False
def install_grammalecte():
"""Install grammalecte CLI."""
log.warning("Missing grammalecte, trying to install it")
# with tempfile.TemporaryDirectory(prefix="padpo_") as tmpdirname:
tmpdirname = "/tmp/_padpo_gramma"
tmpdirname = Path(tmpdirname)
tmpdirname.mkdir(exist_ok=True)
download_request = requests.get(
"https://grammalecte.net/grammalecte/zip/Grammalecte-fr-v1.5.0.zip"
)
download_request.raise_for_status()
zip_file = tmpdirname / "Grammalecte-fr-v1.5.0.zip"
zip_file.write_bytes(download_request.content)
with ZipFile(zip_file, "r") as zip_obj:
zip_obj.extractall(tmpdirname / "Grammalecte-fr-v1.5.0")
subprocess.run(
["pip", "install", str(tmpdirname / "Grammalecte-fr-v1.5.0")]
)

View File

@ -0,0 +1,24 @@
"""Checker for line length."""
from padpo.checkers.baseclass import Checker
from padpo.pofile import PoItem
MAX_LINE_LENGTH = 79
class LineLengthChecker(Checker):
"""Checker for line length."""
def __init__(self):
"""Initialiser."""
super().__init__(name="Line length")
def check_item(self, item: PoItem):
"""Check an item in a `*.po` file."""
for line in item.msgstr:
if len(line) > MAX_LINE_LENGTH - 2: # 2 is for ""
item.add_error(
self.name,
f"Line too long ({len(line) + 2} > "
f"{MAX_LINE_LENGTH}): {line}",
)

47
padpo/checkers/nbsp.py Normal file
View File

@ -0,0 +1,47 @@
"""Checker for missing non breakable spaces."""
import re
from padpo.checkers.baseclass import Checker, replace_quotes
from padpo.pofile import PoItem
class NonBreakableSpaceChecker(Checker):
"""Checker for missing non breakable spaces."""
def __init__(self):
"""Initialiser."""
super().__init__(name="NBSP")
def check_item(self, item: PoItem):
"""Check an item in a `*.po` file."""
text = item.msgstr_rst2txt
for match in re.finditer(r"(.{0,30})(«[^ ])(.{0,30})", text):
self.__add_message(item, *match.groups())
for match in re.finditer(r"(.{0,30})([^ ][»])(.{0,30})", text):
self.__add_message(item, *match.groups())
text = re.sub(r"«\s(.*?)\", replace_quotes, text)
text = re.sub(r"http://", "http-//", text)
text = re.sub(r"https://", "https-//", text)
for sign in "?!:;":
regex = r"(.{0,30})([^ ][" + sign + r"])(.{0,30})"
for match in re.finditer(regex, text):
prefix = item.msgstr_rst2txt[match.start(1) : match.end(1)]
suffix = item.msgstr_rst2txt[match.start(3) : match.end(3)]
match = item.msgstr_rst2txt[match.start(2) : match.end(2)]
self.__add_message_space_before(item, prefix, match, suffix)
def __add_message(self, item, prefix, match, suffix):
item.add_error(
self.name,
"Space should be replaced with a non-breakable space in "
f'"{match}": between ###{prefix}### and ###{suffix}###',
)
def __add_message_space_before(self, item, prefix, match, suffix):
item.add_error(
self.name,
f"There should be a non-breakable space before "
f'"{match[1:]}": between ###{prefix}{match[0]}### and '
f"###{match[1:]}{suffix}###",
)

View File

@ -1,385 +1,29 @@
import argparse
import json
import re
import subprocess
import sys
import tempfile
from abc import ABC, abstractmethod
from pathlib import Path
from typing import List
from zipfile import ZipFile
import requests
import simplelogging
from padpo.pofile import PoFile
from padpo.checkers.doublespace import DoubleSpaceChecker
from padpo.checkers.empty import EmptyChecker
from padpo.checkers.fuzzy import FuzzyChecker
from padpo.checkers.grammalecte import GrammalecteChecker
from padpo.checkers.linelength import LineLengthChecker
from padpo.checkers.nbsp import NonBreakableSpaceChecker
log = None
class PoItem:
def __init__(self, path, lineno):
self.path = path[3:]
self.lineno_start = lineno
self.lineno_end = lineno
self.parsing_msgid = None
self.msgid = []
self.msgstr = []
self.fuzzy = False
self.warnings = []
self.inside_pull_request = False
def append_line(self, line):
self.lineno_end += 1
if line.startswith("msgid"):
self.parsing_msgid = True
self.msgid.append(line[7:-2])
elif line.startswith("msgstr"):
self.parsing_msgid = False
self.msgstr.append(line[8:-2])
elif line.startswith("#, fuzzy"):
self.fuzzy = True
elif line.startswith('"'):
if self.parsing_msgid:
self.msgid.append(line[1:-2])
elif not self.parsing_msgid is None:
self.msgstr.append(line[1:-2])
def __str__(self):
return (
f" - {self.msgid_full_content}\n"
f" => {self.msgstr_full_content}\n"
f" => {self.msgstr_rst2txt}\n"
)
@property
def msgid_full_content(self):
return "".join(self.msgid)
@property
def msgstr_full_content(self):
return "".join(self.msgstr)
@property
def msgid_rst2txt(self):
return self.rst2txt(self.msgid_full_content)
@property
def msgstr_rst2txt(self):
return self.rst2txt(self.msgstr_full_content)
@staticmethod
def rst2txt(text):
text = re.sub(r"::", r":", text)
text = re.sub(r"``(.*?)``", r"« \1 »", text)
text = re.sub(r"\"(.*?)\"", r"« \1 »", text)
text = re.sub(r":pep:`(.*?)`", r"PEP \1", text)
text = re.sub(r":[a-z:]+:`(.+?)`", r"« \1 »", text)
text = re.sub(r"\*\*(.*?)\*\*", r"« \1 »", text)
text = re.sub(
r"\*(.*?)\*", r"« \1 »", text
) # TODO sauf si déjà entre «»
return text
def add_warning(self, checker_name: str, text: str) -> None:
self.warnings.append(Warning(checker_name, text))
def add_error(self, checker_name: str, text: str) -> None:
self.warnings.append(Error(checker_name, text))
class PoFile:
def __init__(self, path=None):
self.content: List[PoItem] = []
self.path = path
if path:
self.parse_file(path)
def parse_file(self, path):
# TODO assert path is a file, not a dir
item = None
with open(path, encoding="utf8") as f:
for lineno, line in enumerate(f):
if line.startswith("#: "):
if item:
self.content.append(item)
item = PoItem(line, lineno + 1)
elif item:
item.append_line(line)
if item:
self.content.append(item)
def __str__(self):
ret = f"Po file: {self.path}\n"
ret += "\n".join(str(item) for item in self.content)
return ret
def rst2txt(self):
return "\n\n".join(item.msgstr_rst2txt for item in self.content)
def display_warnings(self, pull_request_info=None):
self.tag_in_pull_request(pull_request_info)
any_error = False
for item in self.content:
if not item.inside_pull_request:
continue
prefix = f"{self.path}:{item.lineno_start:-4} %s"
log.debug(prefix, "")
for message in item.warnings:
if isinstance(message, Error):
log.error(prefix, message)
any_error = True
elif isinstance(message, Warning):
log.warning(prefix, message)
return any_error
def tag_in_pull_request(self, pull_request_info):
if not pull_request_info:
for item in self.content:
item.inside_pull_request = True
else:
diff = pull_request_info.diff(self.path)
for item in self.content:
item.inside_pull_request = False
for lineno_diff in self.lines_in_diff(diff):
for item in self.content:
if item.lineno_start <= lineno_diff <= item.lineno_end:
item.inside_pull_request = True
@staticmethod
def lines_in_diff(diff):
for line in diff.splitlines():
if line.startswith("@@"):
m = re.search(r"@@\s*\-\d+,\d+\s+\+(\d+),(\d+)\s+@@", line)
if m:
line_start = int(m.group(1))
nb_lines = int(m.group(2))
# github add 3 extra lines around diff info
extra_info_lines = 3
for lineno in range(
line_start + extra_info_lines,
line_start + nb_lines - extra_info_lines,
):
yield lineno
class Message:
def __init__(self, checker_name: str, text: str):
self.checker_name = checker_name
self.text = text
def __str__(self):
return f"[{self.checker_name:^14}] {self.text}"
class Warning(Message):
pass
class Error(Message):
pass
class Checker(ABC):
def __init__(self, name):
self.name = name
def check_file(self, pofile: PoFile):
if not isinstance(pofile, PoFile):
log.error("%s is not an instance of PoFile", str(pofile))
for item in pofile.content:
self.check_item(item)
@abstractmethod
def check_item(self, item: PoItem):
pass
class DoubleSpaceChecker(Checker):
def __init__(self):
super().__init__(name="Double space")
def check_item(self, item: PoItem):
for match in re.finditer(
r"(.{0,30})\s\s(.{0,30})", item.msgstr_full_content
):
item.add_warning(
self.name,
f"Double spaces detected between ###{match.group(1)}### and ###{match.group(2)}###",
)
class LineLengthChecker(Checker):
def __init__(self):
super().__init__(name="Line length")
def check_item(self, item: PoItem):
for line in item.msgstr:
if len(line) > 77: # 77 + 2 ("")
item.add_error(
self.name, f"Line too long ({len(line) + 2} > 79): {line}"
)
class FuzzyChecker(Checker):
def __init__(self):
super().__init__(name="Fuzzy")
def check_item(self, item: PoItem):
if item.fuzzy:
item.add_warning(self.name, "This entry is tagged as fuzzy.")
class EmptyChecker(Checker):
def __init__(self):
super().__init__(name="Empty")
def check_item(self, item: PoItem):
if not item.msgstr_full_content and item.msgid_full_content:
item.add_warning(self.name, "This entry is not translated yet.")
def replace_quotes(match):
length = len(match.group(0)) - 4
return "« " + length * "x" + " »"
class NonBreakableSpaceChecker(Checker):
def __init__(self):
super().__init__(name="NBSP")
def check_item(self, item: PoItem):
text = item.msgstr_rst2txt
for match in re.finditer(r"(.{0,30})(«[^ ])(.{0,30})", text):
self.__add_message(item, *match.groups())
for match in re.finditer(r"(.{0,30})([^ ][»])(.{0,30})", text):
self.__add_message(item, *match.groups())
text = re.sub(r"«\s(.*?)\", replace_quotes, text)
text = re.sub(r"http://", "http-//", text)
text = re.sub(r"https://", "https-//", text)
for sign in "?!:;":
regex = r"(.{0,30})([^ ][" + sign + r"])(.{0,30})"
for match in re.finditer(regex, text):
prefix = item.msgstr_rst2txt[match.start(1) : match.end(1)]
suffix = item.msgstr_rst2txt[match.start(3) : match.end(3)]
match = item.msgstr_rst2txt[match.start(2) : match.end(2)]
self.__add_message_space_before(item, prefix, match, suffix)
def __add_message(self, item, prefix, match, suffix):
item.add_error(
self.name,
f'Space should be replaced with a non-breakable space in "{match}": between ###{prefix}### and ###{suffix}###',
)
def __add_message_space_before(self, item, prefix, match, suffix):
item.add_error(
self.name,
f'There should be a non-breakable space before "{match[1:]}": between ###{prefix}{match[0]}### and ###{match[1:]}{suffix}###',
)
class GrammalecteChecker(Checker):
def __init__(self):
super().__init__(name="Grammalecte")
self.dir = None
def check_file(self, pofile: PoFile):
if not isinstance(pofile, PoFile):
log.error("%s is not an instance of PoFile", str(pofile))
fd, name = tempfile.mkstemp(suffix=".txt", prefix="padpo_", text=True)
with open(name, "w", encoding="utf8") as f:
text = pofile.rst2txt()
text = re.sub(r"«\s(.*?)\", replace_quotes, text)
f.write(text)
try:
result = subprocess.run(
[
"grammalecte-cli.py",
"-f",
name,
"-off",
"apos",
"--json",
"--only_when_errors",
],
capture_output=True,
text=True,
)
except FileNotFoundError as e:
if e.filename == "grammalecte-cli.py":
install_grammalecte()
result = subprocess.run(
[
"grammalecte-cli.py",
"-f",
name,
"-off",
"apos",
"--json",
"--only_when_errors",
],
capture_output=True,
text=True,
)
if result.stdout:
warnings = json.loads(result.stdout)
for warning in warnings["data"]:
for error in warning["lGrammarErrors"]:
if self.filter_out(error):
continue
item_index = int(warning["iParagraph"]) // 2
item = pofile.content[item_index]
start = max(0, int(error["nStart"]) - 40)
end = max(0, int(error["nEnd"]) + 10)
item.add_warning(
self.name,
# self.name + " " + error["sRuleId"], # TODO
error["sMessage"]
+ " => ###"
+ item.msgstr_rst2txt[start:end]
+ "###",
)
Path(name).unlink()
def check_item(self, item: PoItem):
pass
def filter_out(self, error):
msg = error["sRuleId"]
if msg == "esp_milieu_ligne":
return True # double space
if msg == "nbsp_avant_deux_points":
return True
if msg == "nbsp_avant_double_ponctuation":
return True
return False
def install_grammalecte():
log.warning("Missing grammalecte, trying to install it")
# with tempfile.TemporaryDirectory(prefix="padpo_") as tmpdirname:
tmpdirname = "/tmp/_padpo_gramma"
tmpdirname = Path(tmpdirname)
tmpdirname.mkdir(exist_ok=True)
r = requests.get(
"https://grammalecte.net/grammalecte/zip/Grammalecte-fr-v1.5.0.zip"
)
r.raise_for_status()
zip_file = tmpdirname / "Grammalecte-fr-v1.5.0.zip"
zip_file.write_bytes(r.content)
with ZipFile(zip_file, "r") as zip_obj:
zip_obj.extractall(tmpdirname / "Grammalecte-fr-v1.5.0")
subprocess.run(
["pip", "install", str(tmpdirname / "Grammalecte-fr-v1.5.0")]
)
checkers = [
DoubleSpaceChecker(),
LineLengthChecker(),
FuzzyChecker(),
EmptyChecker(),
NonBreakableSpaceChecker(),
FuzzyChecker(),
GrammalecteChecker(),
LineLengthChecker(),
NonBreakableSpaceChecker(),
]

170
padpo/pofile.py Normal file
View File

@ -0,0 +1,170 @@
"""Managment of `*.po` files."""
import re
import simplelogging
log = simplelogging.get_logger()
class PoItem:
def __init__(self, path, lineno):
self.path = path[3:]
self.lineno_start = lineno
self.lineno_end = lineno
self.parsing_msgid = None
self.msgid = []
self.msgstr = []
self.fuzzy = False
self.warnings = []
self.inside_pull_request = False
def append_line(self, line):
self.lineno_end += 1
if line.startswith("msgid"):
self.parsing_msgid = True
self.msgid.append(line[7:-2])
elif line.startswith("msgstr"):
self.parsing_msgid = False
self.msgstr.append(line[8:-2])
elif line.startswith("#, fuzzy"):
self.fuzzy = True
elif line.startswith('"'):
if self.parsing_msgid:
self.msgid.append(line[1:-2])
elif not self.parsing_msgid is None:
self.msgstr.append(line[1:-2])
def __str__(self):
return (
f" - {self.msgid_full_content}\n"
f" => {self.msgstr_full_content}\n"
f" => {self.msgstr_rst2txt}\n"
)
@property
def msgid_full_content(self):
return "".join(self.msgid)
@property
def msgstr_full_content(self):
return "".join(self.msgstr)
@property
def msgid_rst2txt(self):
return self.rst2txt(self.msgid_full_content)
@property
def msgstr_rst2txt(self):
return self.rst2txt(self.msgstr_full_content)
@staticmethod
def rst2txt(text):
text = re.sub(r"::", r":", text)
text = re.sub(r"``(.*?)``", r"« \1 »", text)
text = re.sub(r"\"(.*?)\"", r"« \1 »", text)
text = re.sub(r":pep:`(.*?)`", r"PEP \1", text)
text = re.sub(r":[a-z:]+:`(.+?)`", r"« \1 »", text)
text = re.sub(r"\*\*(.*?)\*\*", r"« \1 »", text)
text = re.sub(
r"\*(.*?)\*", r"« \1 »", text
) # TODO sauf si déjà entre «»
return text
def add_warning(self, checker_name: str, text: str) -> None:
self.warnings.append(Warning(checker_name, text))
def add_error(self, checker_name: str, text: str) -> None:
self.warnings.append(Error(checker_name, text))
class PoFile:
def __init__(self, path=None):
self.content: List[PoItem] = []
self.path = path
if path:
self.parse_file(path)
def parse_file(self, path):
# TODO assert path is a file, not a dir
item = None
with open(path, encoding="utf8") as f:
for lineno, line in enumerate(f):
if line.startswith("#: "):
if item:
self.content.append(item)
item = PoItem(line, lineno + 1)
elif item:
item.append_line(line)
if item:
self.content.append(item)
def __str__(self):
ret = f"Po file: {self.path}\n"
ret += "\n".join(str(item) for item in self.content)
return ret
def rst2txt(self):
return "\n\n".join(item.msgstr_rst2txt for item in self.content)
def display_warnings(self, pull_request_info=None):
self.tag_in_pull_request(pull_request_info)
any_error = False
for item in self.content:
if not item.inside_pull_request:
continue
prefix = f"{self.path}:{item.lineno_start:-4} %s"
log.debug(prefix, "")
for message in item.warnings:
if isinstance(message, Error):
log.error(prefix, message)
any_error = True
elif isinstance(message, Warning):
log.warning(prefix, message)
return any_error
def tag_in_pull_request(self, pull_request_info):
if not pull_request_info:
for item in self.content:
item.inside_pull_request = True
else:
diff = pull_request_info.diff(self.path)
for item in self.content:
item.inside_pull_request = False
for lineno_diff in self.lines_in_diff(diff):
for item in self.content:
if item.lineno_start <= lineno_diff <= item.lineno_end:
item.inside_pull_request = True
@staticmethod
def lines_in_diff(diff):
for line in diff.splitlines():
if line.startswith("@@"):
m = re.search(r"@@\s*\-\d+,\d+\s+\+(\d+),(\d+)\s+@@", line)
if m:
line_start = int(m.group(1))
nb_lines = int(m.group(2))
# github add 3 extra lines around diff info
extra_info_lines = 3
for lineno in range(
line_start + extra_info_lines,
line_start + nb_lines - extra_info_lines,
):
yield lineno
class Message:
def __init__(self, checker_name: str, text: str):
self.checker_name = checker_name
self.text = text
def __str__(self):
return f"[{self.checker_name:^14}] {self.text}"
class Warning(Message):
pass
class Error(Message):
pass

View File

@ -12,7 +12,7 @@ setuptools.setup(
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/vpoulailleau/padpo",
packages=["padpo"],
packages=["padpo", "padpo.checkers"],
package_dir={"padpo": "padpo"},
entry_points={"console_scripts": ["padpo=padpo.padpo:main"]},
install_requires=["simplelogging", "wheel", "requests"],