12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202 |
- """
- babel.messages.frontend
- ~~~~~~~~~~~~~~~~~~~~~~~
- Frontends for the message extraction functionality.
- :copyright: (c) 2013-2025 by the Babel Team.
- :license: BSD, see LICENSE for more details.
- """
- from __future__ import annotations
- import datetime
- import fnmatch
- import logging
- import optparse
- import os
- import re
- import shutil
- import sys
- import tempfile
- import warnings
- from configparser import RawConfigParser
- from io import StringIO
- from typing import BinaryIO, Iterable, Literal
- from babel import Locale, localedata
- from babel import __version__ as VERSION
- from babel.core import UnknownLocaleError
- from babel.messages.catalog import DEFAULT_HEADER, Catalog
- from babel.messages.extract import (
- DEFAULT_KEYWORDS,
- DEFAULT_MAPPING,
- check_and_call_extract_file,
- extract_from_dir,
- )
- from babel.messages.mofile import write_mo
- from babel.messages.pofile import read_po, write_po
- from babel.util import LOCALTZ
- log = logging.getLogger('babel')
- class BaseError(Exception):
- pass
- class OptionError(BaseError):
- pass
- class SetupError(BaseError):
- pass
- class ConfigurationError(BaseError):
- """
- Raised for errors in configuration files.
- """
- def listify_value(arg, split=None):
- """
- Make a list out of an argument.
- Values from `distutils` argument parsing are always single strings;
- values from `optparse` parsing may be lists of strings that may need
- to be further split.
- No matter the input, this function returns a flat list of whitespace-trimmed
- strings, with `None` values filtered out.
- >>> listify_value("foo bar")
- ['foo', 'bar']
- >>> listify_value(["foo bar"])
- ['foo', 'bar']
- >>> listify_value([["foo"], "bar"])
- ['foo', 'bar']
- >>> listify_value([["foo"], ["bar", None, "foo"]])
- ['foo', 'bar', 'foo']
- >>> listify_value("foo, bar, quux", ",")
- ['foo', 'bar', 'quux']
- :param arg: A string or a list of strings
- :param split: The argument to pass to `str.split()`.
- :return:
- """
- out = []
- if not isinstance(arg, (list, tuple)):
- arg = [arg]
- for val in arg:
- if val is None:
- continue
- if isinstance(val, (list, tuple)):
- out.extend(listify_value(val, split=split))
- continue
- out.extend(s.strip() for s in str(val).split(split))
- assert all(isinstance(val, str) for val in out)
- return out
- class CommandMixin:
- # This class is a small shim between Distutils commands and
- # optparse option parsing in the frontend command line.
- #: Option name to be input as `args` on the script command line.
- as_args = None
- #: Options which allow multiple values.
- #: This is used by the `optparse` transmogrification code.
- multiple_value_options = ()
- #: Options which are booleans.
- #: This is used by the `optparse` transmogrification code.
- # (This is actually used by distutils code too, but is never
- # declared in the base class.)
- boolean_options = ()
- #: Option aliases, to retain standalone command compatibility.
- #: Distutils does not support option aliases, but optparse does.
- #: This maps the distutils argument name to an iterable of aliases
- #: that are usable with optparse.
- option_aliases = {}
- #: Choices for options that needed to be restricted to specific
- #: list of choices.
- option_choices = {}
- #: Log object. To allow replacement in the script command line runner.
- log = log
- def __init__(self, dist=None):
- # A less strict version of distutils' `__init__`.
- self.distribution = dist
- self.initialize_options()
- self._dry_run = None
- self.verbose = False
- self.force = None
- self.help = 0
- self.finalized = 0
- def initialize_options(self):
- pass
- def ensure_finalized(self):
- if not self.finalized:
- self.finalize_options()
- self.finalized = 1
- def finalize_options(self):
- raise RuntimeError(
- f"abstract method -- subclass {self.__class__} must override",
- )
- class CompileCatalog(CommandMixin):
- description = 'compile message catalogs to binary MO files'
- user_options = [
- ('domain=', 'D',
- "domains of PO files (space separated list, default 'messages')"),
- ('directory=', 'd',
- 'path to base directory containing the catalogs'),
- ('input-file=', 'i',
- 'name of the input file'),
- ('output-file=', 'o',
- "name of the output file (default "
- "'<output_dir>/<locale>/LC_MESSAGES/<domain>.mo')"),
- ('locale=', 'l',
- 'locale of the catalog to compile'),
- ('use-fuzzy', 'f',
- 'also include fuzzy translations'),
- ('statistics', None,
- 'print statistics about translations'),
- ]
- boolean_options = ['use-fuzzy', 'statistics']
- def initialize_options(self):
- self.domain = 'messages'
- self.directory = None
- self.input_file = None
- self.output_file = None
- self.locale = None
- self.use_fuzzy = False
- self.statistics = False
- def finalize_options(self):
- self.domain = listify_value(self.domain)
- if not self.input_file and not self.directory:
- raise OptionError('you must specify either the input file or the base directory')
- if not self.output_file and not self.directory:
- raise OptionError('you must specify either the output file or the base directory')
- def run(self):
- n_errors = 0
- for domain in self.domain:
- for errors in self._run_domain(domain).values():
- n_errors += len(errors)
- if n_errors:
- self.log.error('%d errors encountered.', n_errors)
- return (1 if n_errors else 0)
- def _run_domain(self, domain):
- po_files = []
- mo_files = []
- if not self.input_file:
- if self.locale:
- po_files.append((self.locale,
- os.path.join(self.directory, self.locale,
- 'LC_MESSAGES',
- f"{domain}.po")))
- mo_files.append(os.path.join(self.directory, self.locale,
- 'LC_MESSAGES',
- f"{domain}.mo"))
- else:
- for locale in os.listdir(self.directory):
- po_file = os.path.join(self.directory, locale,
- 'LC_MESSAGES', f"{domain}.po")
- if os.path.exists(po_file):
- po_files.append((locale, po_file))
- mo_files.append(os.path.join(self.directory, locale,
- 'LC_MESSAGES',
- f"{domain}.mo"))
- else:
- po_files.append((self.locale, self.input_file))
- if self.output_file:
- mo_files.append(self.output_file)
- else:
- mo_files.append(os.path.join(self.directory, self.locale,
- 'LC_MESSAGES',
- f"{domain}.mo"))
- if not po_files:
- raise OptionError('no message catalogs found')
- catalogs_and_errors = {}
- for idx, (locale, po_file) in enumerate(po_files):
- mo_file = mo_files[idx]
- with open(po_file, 'rb') as infile:
- catalog = read_po(infile, locale)
- if self.statistics:
- translated = 0
- for message in list(catalog)[1:]:
- if message.string:
- translated += 1
- percentage = 0
- if len(catalog):
- percentage = translated * 100 // len(catalog)
- self.log.info(
- '%d of %d messages (%d%%) translated in %s',
- translated, len(catalog), percentage, po_file,
- )
- if catalog.fuzzy and not self.use_fuzzy:
- self.log.info('catalog %s is marked as fuzzy, skipping', po_file)
- continue
- catalogs_and_errors[catalog] = catalog_errors = list(catalog.check())
- for message, errors in catalog_errors:
- for error in errors:
- self.log.error(
- 'error: %s:%d: %s', po_file, message.lineno, error,
- )
- self.log.info('compiling catalog %s to %s', po_file, mo_file)
- with open(mo_file, 'wb') as outfile:
- write_mo(outfile, catalog, use_fuzzy=self.use_fuzzy)
- return catalogs_and_errors
- def _make_directory_filter(ignore_patterns):
- """
- Build a directory_filter function based on a list of ignore patterns.
- """
- def cli_directory_filter(dirname):
- basename = os.path.basename(dirname)
- return not any(
- fnmatch.fnmatch(basename, ignore_pattern)
- for ignore_pattern
- in ignore_patterns
- )
- return cli_directory_filter
- class ExtractMessages(CommandMixin):
- description = 'extract localizable strings from the project code'
- user_options = [
- ('charset=', None,
- 'charset to use in the output file (default "utf-8")'),
- ('keywords=', 'k',
- 'space-separated list of keywords to look for in addition to the '
- 'defaults (may be repeated multiple times)'),
- ('no-default-keywords', None,
- 'do not include the default keywords'),
- ('mapping-file=', 'F',
- 'path to the mapping configuration file'),
- ('no-location', None,
- 'do not include location comments with filename and line number'),
- ('add-location=', None,
- 'location lines format. If it is not given or "full", it generates '
- 'the lines with both file name and line number. If it is "file", '
- 'the line number part is omitted. If it is "never", it completely '
- 'suppresses the lines (same as --no-location).'),
- ('omit-header', None,
- 'do not include msgid "" entry in header'),
- ('output-file=', 'o',
- 'name of the output file'),
- ('width=', 'w',
- 'set output line width (default 76)'),
- ('no-wrap', None,
- 'do not break long message lines, longer than the output line width, '
- 'into several lines'),
- ('sort-output', None,
- 'generate sorted output (default False)'),
- ('sort-by-file', None,
- 'sort output by file location (default False)'),
- ('msgid-bugs-address=', None,
- 'set report address for msgid'),
- ('copyright-holder=', None,
- 'set copyright holder in output'),
- ('project=', None,
- 'set project name in output'),
- ('version=', None,
- 'set project version in output'),
- ('add-comments=', 'c',
- 'place comment block with TAG (or those preceding keyword lines) in '
- 'output file. Separate multiple TAGs with commas(,)'), # TODO: Support repetition of this argument
- ('strip-comments', 's',
- 'strip the comment TAGs from the comments.'),
- ('input-paths=', None,
- 'files or directories that should be scanned for messages. Separate multiple '
- 'files or directories with commas(,)'), # TODO: Support repetition of this argument
- ('input-dirs=', None, # TODO (3.x): Remove me.
- 'alias for input-paths (does allow files as well as directories).'),
- ('ignore-dirs=', None,
- 'Patterns for directories to ignore when scanning for messages. '
- 'Separate multiple patterns with spaces (default ".* ._")'),
- ('header-comment=', None,
- 'header comment for the catalog'),
- ('last-translator=', None,
- 'set the name and email of the last translator in output'),
- ]
- boolean_options = [
- 'no-default-keywords', 'no-location', 'omit-header', 'no-wrap',
- 'sort-output', 'sort-by-file', 'strip-comments',
- ]
- as_args = 'input-paths'
- multiple_value_options = (
- 'add-comments',
- 'keywords',
- 'ignore-dirs',
- )
- option_aliases = {
- 'keywords': ('--keyword',),
- 'mapping-file': ('--mapping',),
- 'output-file': ('--output',),
- 'strip-comments': ('--strip-comment-tags',),
- 'last-translator': ('--last-translator',),
- }
- option_choices = {
- 'add-location': ('full', 'file', 'never'),
- }
- def initialize_options(self):
- self.charset = 'utf-8'
- self.keywords = None
- self.no_default_keywords = False
- self.mapping_file = None
- self.no_location = False
- self.add_location = None
- self.omit_header = False
- self.output_file = None
- self.input_dirs = None
- self.input_paths = None
- self.width = None
- self.no_wrap = False
- self.sort_output = False
- self.sort_by_file = False
- self.msgid_bugs_address = None
- self.copyright_holder = None
- self.project = None
- self.version = None
- self.add_comments = None
- self.strip_comments = False
- self.include_lineno = True
- self.ignore_dirs = None
- self.header_comment = None
- self.last_translator = None
- def finalize_options(self):
- if self.input_dirs:
- if not self.input_paths:
- self.input_paths = self.input_dirs
- else:
- raise OptionError(
- 'input-dirs and input-paths are mutually exclusive',
- )
- keywords = {} if self.no_default_keywords else DEFAULT_KEYWORDS.copy()
- keywords.update(parse_keywords(listify_value(self.keywords)))
- self.keywords = keywords
- if not self.keywords:
- raise OptionError(
- 'you must specify new keywords if you disable the default ones',
- )
- if not self.output_file:
- raise OptionError('no output file specified')
- if self.no_wrap and self.width:
- raise OptionError(
- "'--no-wrap' and '--width' are mutually exclusive",
- )
- if not self.no_wrap and not self.width:
- self.width = 76
- elif self.width is not None:
- self.width = int(self.width)
- if self.sort_output and self.sort_by_file:
- raise OptionError(
- "'--sort-output' and '--sort-by-file' are mutually exclusive",
- )
- if self.input_paths:
- if isinstance(self.input_paths, str):
- self.input_paths = re.split(r',\s*', self.input_paths)
- elif self.distribution is not None:
- self.input_paths = dict.fromkeys([
- k.split('.', 1)[0]
- for k in (self.distribution.packages or ())
- ]).keys()
- else:
- self.input_paths = []
- if not self.input_paths:
- raise OptionError("no input files or directories specified")
- for path in self.input_paths:
- if not os.path.exists(path):
- raise OptionError(f"Input path: {path} does not exist")
- self.add_comments = listify_value(self.add_comments or (), ",")
- if self.distribution:
- if not self.project:
- self.project = self.distribution.get_name()
- if not self.version:
- self.version = self.distribution.get_version()
- if self.add_location == 'never':
- self.no_location = True
- elif self.add_location == 'file':
- self.include_lineno = False
- ignore_dirs = listify_value(self.ignore_dirs)
- if ignore_dirs:
- self.directory_filter = _make_directory_filter(ignore_dirs)
- else:
- self.directory_filter = None
- def _build_callback(self, path: str):
- def callback(filename: str, method: str, options: dict):
- if method == 'ignore':
- return
- # If we explicitly provide a full filepath, just use that.
- # Otherwise, path will be the directory path and filename
- # is the relative path from that dir to the file.
- # So we can join those to get the full filepath.
- if os.path.isfile(path):
- filepath = path
- else:
- filepath = os.path.normpath(os.path.join(path, filename))
- optstr = ''
- if options:
- opt_values = ", ".join(f'{k}="{v}"' for k, v in options.items())
- optstr = f" ({opt_values})"
- self.log.info('extracting messages from %s%s', filepath, optstr)
- return callback
- def run(self):
- mappings = self._get_mappings()
- with open(self.output_file, 'wb') as outfile:
- catalog = Catalog(project=self.project,
- version=self.version,
- msgid_bugs_address=self.msgid_bugs_address,
- copyright_holder=self.copyright_holder,
- charset=self.charset,
- header_comment=(self.header_comment or DEFAULT_HEADER),
- last_translator=self.last_translator)
- for path, method_map, options_map in mappings:
- callback = self._build_callback(path)
- if os.path.isfile(path):
- current_dir = os.getcwd()
- extracted = check_and_call_extract_file(
- path, method_map, options_map,
- callback, self.keywords, self.add_comments,
- self.strip_comments, current_dir,
- )
- else:
- extracted = extract_from_dir(
- path, method_map, options_map,
- keywords=self.keywords,
- comment_tags=self.add_comments,
- callback=callback,
- strip_comment_tags=self.strip_comments,
- directory_filter=self.directory_filter,
- )
- for filename, lineno, message, comments, context in extracted:
- if os.path.isfile(path):
- filepath = filename # already normalized
- else:
- filepath = os.path.normpath(os.path.join(path, filename))
- catalog.add(message, None, [(filepath, lineno)],
- auto_comments=comments, context=context)
- self.log.info('writing PO template file to %s', self.output_file)
- write_po(outfile, catalog, width=self.width,
- no_location=self.no_location,
- omit_header=self.omit_header,
- sort_output=self.sort_output,
- sort_by_file=self.sort_by_file,
- include_lineno=self.include_lineno)
- def _get_mappings(self):
- mappings = []
- if self.mapping_file:
- if self.mapping_file.endswith(".toml"):
- with open(self.mapping_file, "rb") as fileobj:
- file_style = (
- "pyproject.toml"
- if os.path.basename(self.mapping_file) == "pyproject.toml"
- else "standalone"
- )
- method_map, options_map = _parse_mapping_toml(
- fileobj,
- filename=self.mapping_file,
- style=file_style,
- )
- else:
- with open(self.mapping_file) as fileobj:
- method_map, options_map = parse_mapping_cfg(fileobj, filename=self.mapping_file)
- for path in self.input_paths:
- mappings.append((path, method_map, options_map))
- elif getattr(self.distribution, 'message_extractors', None):
- message_extractors = self.distribution.message_extractors
- for path, mapping in message_extractors.items():
- if isinstance(mapping, str):
- method_map, options_map = parse_mapping_cfg(StringIO(mapping))
- else:
- method_map, options_map = [], {}
- for pattern, method, options in mapping:
- method_map.append((pattern, method))
- options_map[pattern] = options or {}
- mappings.append((path, method_map, options_map))
- else:
- for path in self.input_paths:
- mappings.append((path, DEFAULT_MAPPING, {}))
- return mappings
- class InitCatalog(CommandMixin):
- description = 'create a new catalog based on a POT file'
- user_options = [
- ('domain=', 'D',
- "domain of PO file (default 'messages')"),
- ('input-file=', 'i',
- 'name of the input file'),
- ('output-dir=', 'd',
- 'path to output directory'),
- ('output-file=', 'o',
- "name of the output file (default "
- "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"),
- ('locale=', 'l',
- 'locale for the new localized catalog'),
- ('width=', 'w',
- 'set output line width (default 76)'),
- ('no-wrap', None,
- 'do not break long message lines, longer than the output line width, '
- 'into several lines'),
- ]
- boolean_options = ['no-wrap']
- def initialize_options(self):
- self.output_dir = None
- self.output_file = None
- self.input_file = None
- self.locale = None
- self.domain = 'messages'
- self.no_wrap = False
- self.width = None
- def finalize_options(self):
- if not self.input_file:
- raise OptionError('you must specify the input file')
- if not self.locale:
- raise OptionError('you must provide a locale for the new catalog')
- try:
- self._locale = Locale.parse(self.locale)
- except UnknownLocaleError as e:
- raise OptionError(e) from e
- if not self.output_file and not self.output_dir:
- raise OptionError('you must specify the output directory')
- if not self.output_file:
- self.output_file = os.path.join(self.output_dir, self.locale,
- 'LC_MESSAGES', f"{self.domain}.po")
- if not os.path.exists(os.path.dirname(self.output_file)):
- os.makedirs(os.path.dirname(self.output_file))
- if self.no_wrap and self.width:
- raise OptionError("'--no-wrap' and '--width' are mutually exclusive")
- if not self.no_wrap and not self.width:
- self.width = 76
- elif self.width is not None:
- self.width = int(self.width)
- def run(self):
- self.log.info(
- 'creating catalog %s based on %s', self.output_file, self.input_file,
- )
- with open(self.input_file, 'rb') as infile:
- # Although reading from the catalog template, read_po must be fed
- # the locale in order to correctly calculate plurals
- catalog = read_po(infile, locale=self.locale)
- catalog.locale = self._locale
- catalog.revision_date = datetime.datetime.now(LOCALTZ)
- catalog.fuzzy = False
- with open(self.output_file, 'wb') as outfile:
- write_po(outfile, catalog, width=self.width)
- class UpdateCatalog(CommandMixin):
- description = 'update message catalogs from a POT file'
- user_options = [
- ('domain=', 'D',
- "domain of PO file (default 'messages')"),
- ('input-file=', 'i',
- 'name of the input file'),
- ('output-dir=', 'd',
- 'path to base directory containing the catalogs'),
- ('output-file=', 'o',
- "name of the output file (default "
- "'<output_dir>/<locale>/LC_MESSAGES/<domain>.po')"),
- ('omit-header', None,
- "do not include msgid "" entry in header"),
- ('locale=', 'l',
- 'locale of the catalog to compile'),
- ('width=', 'w',
- 'set output line width (default 76)'),
- ('no-wrap', None,
- 'do not break long message lines, longer than the output line width, '
- 'into several lines'),
- ('ignore-obsolete=', None,
- 'whether to omit obsolete messages from the output'),
- ('init-missing=', None,
- 'if any output files are missing, initialize them first'),
- ('no-fuzzy-matching', 'N',
- 'do not use fuzzy matching'),
- ('update-header-comment', None,
- 'update target header comment'),
- ('previous', None,
- 'keep previous msgids of translated messages'),
- ('check=', None,
- 'don\'t update the catalog, just return the status. Return code 0 '
- 'means nothing would change. Return code 1 means that the catalog '
- 'would be updated'),
- ('ignore-pot-creation-date=', None,
- 'ignore changes to POT-Creation-Date when updating or checking'),
- ]
- boolean_options = [
- 'omit-header', 'no-wrap', 'ignore-obsolete', 'init-missing',
- 'no-fuzzy-matching', 'previous', 'update-header-comment',
- 'check', 'ignore-pot-creation-date',
- ]
- def initialize_options(self):
- self.domain = 'messages'
- self.input_file = None
- self.output_dir = None
- self.output_file = None
- self.omit_header = False
- self.locale = None
- self.width = None
- self.no_wrap = False
- self.ignore_obsolete = False
- self.init_missing = False
- self.no_fuzzy_matching = False
- self.update_header_comment = False
- self.previous = False
- self.check = False
- self.ignore_pot_creation_date = False
- def finalize_options(self):
- if not self.input_file:
- raise OptionError('you must specify the input file')
- if not self.output_file and not self.output_dir:
- raise OptionError('you must specify the output file or directory')
- if self.output_file and not self.locale:
- raise OptionError('you must specify the locale')
- if self.init_missing:
- if not self.locale:
- raise OptionError(
- 'you must specify the locale for '
- 'the init-missing option to work',
- )
- try:
- self._locale = Locale.parse(self.locale)
- except UnknownLocaleError as e:
- raise OptionError(e) from e
- else:
- self._locale = None
- if self.no_wrap and self.width:
- raise OptionError("'--no-wrap' and '--width' are mutually exclusive")
- if not self.no_wrap and not self.width:
- self.width = 76
- elif self.width is not None:
- self.width = int(self.width)
- if self.no_fuzzy_matching and self.previous:
- self.previous = False
- def run(self):
- check_status = {}
- po_files = []
- if not self.output_file:
- if self.locale:
- po_files.append((self.locale,
- os.path.join(self.output_dir, self.locale,
- 'LC_MESSAGES',
- f"{self.domain}.po")))
- else:
- for locale in os.listdir(self.output_dir):
- po_file = os.path.join(self.output_dir, locale,
- 'LC_MESSAGES',
- f"{self.domain}.po")
- if os.path.exists(po_file):
- po_files.append((locale, po_file))
- else:
- po_files.append((self.locale, self.output_file))
- if not po_files:
- raise OptionError('no message catalogs found')
- domain = self.domain
- if not domain:
- domain = os.path.splitext(os.path.basename(self.input_file))[0]
- with open(self.input_file, 'rb') as infile:
- template = read_po(infile)
- for locale, filename in po_files:
- if self.init_missing and not os.path.exists(filename):
- if self.check:
- check_status[filename] = False
- continue
- self.log.info(
- 'creating catalog %s based on %s', filename, self.input_file,
- )
- with open(self.input_file, 'rb') as infile:
- # Although reading from the catalog template, read_po must
- # be fed the locale in order to correctly calculate plurals
- catalog = read_po(infile, locale=self.locale)
- catalog.locale = self._locale
- catalog.revision_date = datetime.datetime.now(LOCALTZ)
- catalog.fuzzy = False
- with open(filename, 'wb') as outfile:
- write_po(outfile, catalog)
- self.log.info('updating catalog %s based on %s', filename, self.input_file)
- with open(filename, 'rb') as infile:
- catalog = read_po(infile, locale=locale, domain=domain)
- catalog.update(
- template, self.no_fuzzy_matching,
- update_header_comment=self.update_header_comment,
- update_creation_date=not self.ignore_pot_creation_date,
- )
- tmpname = os.path.join(os.path.dirname(filename),
- tempfile.gettempprefix() +
- os.path.basename(filename))
- try:
- with open(tmpname, 'wb') as tmpfile:
- write_po(tmpfile, catalog,
- omit_header=self.omit_header,
- ignore_obsolete=self.ignore_obsolete,
- include_previous=self.previous, width=self.width)
- except Exception:
- os.remove(tmpname)
- raise
- if self.check:
- with open(filename, "rb") as origfile:
- original_catalog = read_po(origfile)
- with open(tmpname, "rb") as newfile:
- updated_catalog = read_po(newfile)
- updated_catalog.revision_date = original_catalog.revision_date
- check_status[filename] = updated_catalog.is_identical(original_catalog)
- os.remove(tmpname)
- continue
- try:
- os.rename(tmpname, filename)
- except OSError:
- # We're probably on Windows, which doesn't support atomic
- # renames, at least not through Python
- # If the error is in fact due to a permissions problem, that
- # same error is going to be raised from one of the following
- # operations
- os.remove(filename)
- shutil.copy(tmpname, filename)
- os.remove(tmpname)
- if self.check:
- for filename, up_to_date in check_status.items():
- if up_to_date:
- self.log.info('Catalog %s is up to date.', filename)
- else:
- self.log.warning('Catalog %s is out of date.', filename)
- if not all(check_status.values()):
- raise BaseError("Some catalogs are out of date.")
- else:
- self.log.info("All the catalogs are up-to-date.")
- return
- class CommandLineInterface:
- """Command-line interface.
- This class provides a simple command-line interface to the message
- extraction and PO file generation functionality.
- """
- usage = '%%prog %s [options] %s'
- version = f'%prog {VERSION}'
- commands = {
- 'compile': 'compile message catalogs to MO files',
- 'extract': 'extract messages from source files and generate a POT file',
- 'init': 'create new message catalogs from a POT file',
- 'update': 'update existing message catalogs from a POT file',
- }
- command_classes = {
- 'compile': CompileCatalog,
- 'extract': ExtractMessages,
- 'init': InitCatalog,
- 'update': UpdateCatalog,
- }
- log = None # Replaced on instance level
- def run(self, argv=None):
- """Main entry point of the command-line interface.
- :param argv: list of arguments passed on the command-line
- """
- if argv is None:
- argv = sys.argv
- self.parser = optparse.OptionParser(usage=self.usage % ('command', '[args]'),
- version=self.version)
- self.parser.disable_interspersed_args()
- self.parser.print_help = self._help
- self.parser.add_option('--list-locales', dest='list_locales',
- action='store_true',
- help="print all known locales and exit")
- self.parser.add_option('-v', '--verbose', action='store_const',
- dest='loglevel', const=logging.DEBUG,
- help='print as much as possible')
- self.parser.add_option('-q', '--quiet', action='store_const',
- dest='loglevel', const=logging.ERROR,
- help='print as little as possible')
- self.parser.set_defaults(list_locales=False, loglevel=logging.INFO)
- options, args = self.parser.parse_args(argv[1:])
- self._configure_logging(options.loglevel)
- if options.list_locales:
- identifiers = localedata.locale_identifiers()
- id_width = max(len(identifier) for identifier in identifiers) + 1
- for identifier in sorted(identifiers):
- locale = Locale.parse(identifier)
- print(f"{identifier:<{id_width}} {locale.english_name}")
- return 0
- if not args:
- self.parser.error('no valid command or option passed. '
- 'Try the -h/--help option for more information.')
- cmdname = args[0]
- if cmdname not in self.commands:
- self.parser.error(f'unknown command "{cmdname}"')
- cmdinst = self._configure_command(cmdname, args[1:])
- return cmdinst.run()
- def _configure_logging(self, loglevel):
- self.log = log
- self.log.setLevel(loglevel)
- # Don't add a new handler for every instance initialization (#227), this
- # would cause duplicated output when the CommandLineInterface as an
- # normal Python class.
- if self.log.handlers:
- handler = self.log.handlers[0]
- else:
- handler = logging.StreamHandler()
- self.log.addHandler(handler)
- handler.setLevel(loglevel)
- formatter = logging.Formatter('%(message)s')
- handler.setFormatter(formatter)
- def _help(self):
- print(self.parser.format_help())
- print("commands:")
- cmd_width = max(8, max(len(command) for command in self.commands) + 1)
- for name, description in sorted(self.commands.items()):
- print(f" {name:<{cmd_width}} {description}")
- def _configure_command(self, cmdname, argv):
- """
- :type cmdname: str
- :type argv: list[str]
- """
- cmdclass = self.command_classes[cmdname]
- cmdinst = cmdclass()
- if self.log:
- cmdinst.log = self.log # Use our logger, not distutils'.
- assert isinstance(cmdinst, CommandMixin)
- cmdinst.initialize_options()
- parser = optparse.OptionParser(
- usage=self.usage % (cmdname, ''),
- description=self.commands[cmdname],
- )
- as_args: str | None = getattr(cmdclass, "as_args", None)
- for long, short, help in cmdclass.user_options:
- name = long.strip("=")
- default = getattr(cmdinst, name.replace("-", "_"))
- strs = [f"--{name}"]
- if short:
- strs.append(f"-{short}")
- strs.extend(cmdclass.option_aliases.get(name, ()))
- choices = cmdclass.option_choices.get(name, None)
- if name == as_args:
- parser.usage += f"<{name}>"
- elif name in cmdclass.boolean_options:
- parser.add_option(*strs, action="store_true", help=help)
- elif name in cmdclass.multiple_value_options:
- parser.add_option(*strs, action="append", help=help, choices=choices)
- else:
- parser.add_option(*strs, help=help, default=default, choices=choices)
- options, args = parser.parse_args(argv)
- if as_args:
- setattr(options, as_args.replace('-', '_'), args)
- for key, value in vars(options).items():
- setattr(cmdinst, key, value)
- try:
- cmdinst.ensure_finalized()
- except OptionError as err:
- parser.error(str(err))
- return cmdinst
- def main():
- return CommandLineInterface().run(sys.argv)
- def parse_mapping(fileobj, filename=None):
- warnings.warn(
- "parse_mapping is deprecated, use parse_mapping_cfg instead",
- DeprecationWarning,
- stacklevel=2,
- )
- return parse_mapping_cfg(fileobj, filename)
- def parse_mapping_cfg(fileobj, filename=None):
- """Parse an extraction method mapping from a file-like object.
- :param fileobj: a readable file-like object containing the configuration
- text to parse
- :param filename: the name of the file being parsed, for error messages
- """
- extractors = {}
- method_map = []
- options_map = {}
- parser = RawConfigParser()
- parser.read_file(fileobj, filename)
- for section in parser.sections():
- if section == 'extractors':
- extractors = dict(parser.items(section))
- else:
- method, pattern = (part.strip() for part in section.split(':', 1))
- method_map.append((pattern, method))
- options_map[pattern] = dict(parser.items(section))
- if extractors:
- for idx, (pattern, method) in enumerate(method_map):
- if method in extractors:
- method = extractors[method]
- method_map[idx] = (pattern, method)
- return method_map, options_map
- def _parse_config_object(config: dict, *, filename="(unknown)"):
- extractors = {}
- method_map = []
- options_map = {}
- extractors_read = config.get("extractors", {})
- if not isinstance(extractors_read, dict):
- raise ConfigurationError(f"{filename}: extractors: Expected a dictionary, got {type(extractors_read)!r}")
- for method, callable_spec in extractors_read.items():
- if not isinstance(method, str):
- # Impossible via TOML, but could happen with a custom object.
- raise ConfigurationError(f"{filename}: extractors: Extraction method must be a string, got {method!r}")
- if not isinstance(callable_spec, str):
- raise ConfigurationError(f"{filename}: extractors: Callable specification must be a string, got {callable_spec!r}")
- extractors[method] = callable_spec
- if "mapping" in config:
- raise ConfigurationError(f"{filename}: 'mapping' is not a valid key, did you mean 'mappings'?")
- mappings_read = config.get("mappings", [])
- if not isinstance(mappings_read, list):
- raise ConfigurationError(f"{filename}: mappings: Expected a list, got {type(mappings_read)!r}")
- for idx, entry in enumerate(mappings_read):
- if not isinstance(entry, dict):
- raise ConfigurationError(f"{filename}: mappings[{idx}]: Expected a dictionary, got {type(entry)!r}")
- entry = entry.copy()
- method = entry.pop("method", None)
- if not isinstance(method, str):
- raise ConfigurationError(f"{filename}: mappings[{idx}]: 'method' must be a string, got {method!r}")
- method = extractors.get(method, method) # Map the extractor name to the callable now
- pattern = entry.pop("pattern", None)
- if not isinstance(pattern, (list, str)):
- raise ConfigurationError(f"{filename}: mappings[{idx}]: 'pattern' must be a list or a string, got {pattern!r}")
- if not isinstance(pattern, list):
- pattern = [pattern]
- for pat in pattern:
- if not isinstance(pat, str):
- raise ConfigurationError(f"{filename}: mappings[{idx}]: 'pattern' elements must be strings, got {pat!r}")
- method_map.append((pat, method))
- options_map[pat] = entry
- return method_map, options_map
- def _parse_mapping_toml(
- fileobj: BinaryIO,
- filename: str = "(unknown)",
- style: Literal["standalone", "pyproject.toml"] = "standalone",
- ):
- """Parse an extraction method mapping from a binary file-like object.
- .. warning: As of this version of Babel, this is a private API subject to changes.
- :param fileobj: a readable binary file-like object containing the configuration TOML to parse
- :param filename: the name of the file being parsed, for error messages
- :param style: whether the file is in the style of a `pyproject.toml` file, i.e. whether to look for `tool.babel`.
- """
- try:
- import tomllib
- except ImportError:
- try:
- import tomli as tomllib
- except ImportError as ie: # pragma: no cover
- raise ImportError("tomli or tomllib is required to parse TOML files") from ie
- try:
- parsed_data = tomllib.load(fileobj)
- except tomllib.TOMLDecodeError as e:
- raise ConfigurationError(f"{filename}: Error parsing TOML file: {e}") from e
- if style == "pyproject.toml":
- try:
- babel_data = parsed_data["tool"]["babel"]
- except (TypeError, KeyError) as e:
- raise ConfigurationError(f"{filename}: No 'tool.babel' section found in file") from e
- elif style == "standalone":
- babel_data = parsed_data
- if "babel" in babel_data:
- raise ConfigurationError(f"{filename}: 'babel' should not be present in a stand-alone configuration file")
- else: # pragma: no cover
- raise ValueError(f"Unknown TOML style {style!r}")
- return _parse_config_object(babel_data, filename=filename)
- def _parse_spec(s: str) -> tuple[int | None, tuple[int | tuple[int, str], ...]]:
- inds = []
- number = None
- for x in s.split(','):
- if x[-1] == 't':
- number = int(x[:-1])
- elif x[-1] == 'c':
- inds.append((int(x[:-1]), 'c'))
- else:
- inds.append(int(x))
- return number, tuple(inds)
- def parse_keywords(strings: Iterable[str] = ()):
- """Parse keywords specifications from the given list of strings.
- >>> import pprint
- >>> keywords = ['_', 'dgettext:2', 'dngettext:2,3', 'pgettext:1c,2',
- ... 'polymorphic:1', 'polymorphic:2,2t', 'polymorphic:3c,3t']
- >>> pprint.pprint(parse_keywords(keywords))
- {'_': None,
- 'dgettext': (2,),
- 'dngettext': (2, 3),
- 'pgettext': ((1, 'c'), 2),
- 'polymorphic': {None: (1,), 2: (2,), 3: ((3, 'c'),)}}
- The input keywords are in GNU Gettext style; see :doc:`cmdline` for details.
- The output is a dictionary mapping keyword names to a dictionary of specifications.
- Keys in this dictionary are numbers of arguments, where ``None`` means that all numbers
- of arguments are matched, and a number means only calls with that number of arguments
- are matched (which happens when using the "t" specifier). However, as a special
- case for backwards compatibility, if the dictionary of specifications would
- be ``{None: x}``, i.e., there is only one specification and it matches all argument
- counts, then it is collapsed into just ``x``.
- A specification is either a tuple or None. If a tuple, each element can be either a number
- ``n``, meaning that the nth argument should be extracted as a message, or the tuple
- ``(n, 'c')``, meaning that the nth argument should be extracted as context for the
- messages. A ``None`` specification is equivalent to ``(1,)``, extracting the first
- argument.
- """
- keywords = {}
- for string in strings:
- if ':' in string:
- funcname, spec_str = string.split(':')
- number, spec = _parse_spec(spec_str)
- else:
- funcname = string
- number = None
- spec = None
- keywords.setdefault(funcname, {})[number] = spec
- # For best backwards compatibility, collapse {None: x} into x.
- for k, v in keywords.items():
- if set(v) == {None}:
- keywords[k] = v[None]
- return keywords
- def __getattr__(name: str):
- # Re-exports for backwards compatibility;
- # `setuptools_frontend` is the canonical import location.
- if name in {'check_message_extractors', 'compile_catalog', 'extract_messages', 'init_catalog', 'update_catalog'}:
- from babel.messages import setuptools_frontend
- return getattr(setuptools_frontend, name)
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
- if __name__ == '__main__':
- main()
|