CI/CD パイプラインで Amazon Translate を使用してウェブサイトまたはアプリケーションを自動的に翻訳する

<p>Hello</p>

<p>{{ gettext(‘Hello’) }}</p>

<p>{{ _(‘Hello’) }}</p>

#: file.html:23
msgid “Hello”
msgstr “Hola”

$ pybabel extract -F babel.cfg -o messages.pot .

$ pybabel init -i messages.pot -d app/translations -l es

#!/usr/bin/env python
import argparse
import boto3
import os
import polib
import time

DEFAULT_SOURCE_LANG = 'en'
DEFAULT_TRANSLATIONS_ROOT_DIR = 'app/translations'

def validate_translations_dir(dir):
    """Checks that the given translations directory exists.

    Args:
        dir: The relative path to the directory.

    Raises:
        Exception: The translations directory does not exist.
    """
    if not os.path.exists(dir):
        raise Exception("Translations directory '{}' does not exist".format(dir))

def get_pofile_path_for_language(lang, translations_dir):
    """Returns the relative path to the .PO file for the specified language.

    Args:
        lang: The language code.
        translations_dir: The relative path to the directory containing all
            translations-related files.

    Returns:
        A string with the relative path to the .PO file.
    """
    return "{}/{}/LC_MESSAGES/messages.po".format(translations_dir, lang)

def get_target_languages(translations_dir):
    """Returns all languages that the app should be translated into.

    Args:
        translations_dir: The relative path to the directory containing all
            translations-related files.

    Returns:
        A list of language codes.For example:

        ['es', 'fr']
    """
    target_languages = next(os.walk(translations_dir))[1]
    print('Detected languages:', target_languages)
    return target_languages

def translate_language(src_lang, dest_lang, pofile_path):
    """Translate the app strings from and to the specified languages by
    invoking the Amazon Translate API.

    When this function completes, the translated strings will be written to the
    .PO file of the destination language.

    Args:
        src_lang: The ISO code of the language to translate from, e.g., 'en'.
        dest_lang: The ISO code of the language to translate to, e.g., 'es'.
        pofile_path: The path to the .PO file containing the strings to be
            translated.
    """
    print("Translating from '{}' to '{}'".format(src_lang, dest_lang))

    translate = boto3.client('translate')

    po = polib.pofile(pofile_path)
    for entry in po:
        print("Translating entry '{}' to language '{}'".format(entry.msgid, dest_lang))

        # Calculate the time that the request takes
        time_before = time.time_ns()
        response = translate.translate_text(
            Text=entry.msgid,
            SourceLanguageCode=src_lang,
            TargetLanguageCode=dest_lang
        )
        time_after = time.time_ns()

        entry.msgstr = response['TranslatedText']

        # Wait if needed to avoid throtling
        time_diff = time_after - time_before
        if time_diff < 50000:
            time.sleep(0.05)

    po.save(pofile_path)

def generate_translations_without_cache(src_lang, translations_dir):
    """Translate the app to all applicable target languages, without using
    a cache.

    Args:
        src_lang: The ISO code of the language to translate from, e.g., 'en'.
        translations_dir: The relative path to the directory containing all
            translations-related files.
    """
    target_languages = get_target_languages(translations_dir)

    for dest_lang in target_languages:
        pofile_path = get_pofile_path_for_language(dest_lang, translations_dir)
        translate_language(src_lang, dest_lang, pofile_path)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--source-language",
                        help="the ISO code of the source language to translate from (e.g., 'en'), defaults to '{}'".format(DEFAULT_SOURCE_LANG),
                        default=DEFAULT_SOURCE_LANG)
    parser.add_argument("--translations-dir",
                        help="the relative path to the directory containing all translation data, defaults to '{}'".format(DEFAULT_TRANSLATIONS_ROOT_DIR),
                        default=DEFAULT_TRANSLATIONS_ROOT_DIR)
    args = parser.parse_args()

    print("Source language is:      {}".format(args.source_language))
    print("Translations dir is:     {}".format(args.translations_dir))

    # Check that the translations folder exists.
    validate_translations_dir(args.translations_dir)

    generate_translations_without_cache(args.source_language, args.translations_dir)

$ ./generate_translations.py
Source language is:      en
Translations dir is:     app/translations
Detected languages: ['es']
Translating entry 'Hello' to language 'es'

$ pybabel compile -d app/translations/

version: 0.2

phases:
  install:
    runtime-versions:
      python: 3.7
  pre_build:
    commands:
      - pip install -r requirements.txt
  build:
    commands:
      # Extract translation strings from our application code
      - pybabel extract --omit-header -F babel.cfg -o messages.pot .
      # Generate one translation file per applicable language
      - pybabel init -i messages.pot -d app/translations -l es
      # Run our automated translation script
      - ./generate_translations.py
      # Finally, compile all translations so that they can be used by our app
      - pybabel compile -d app/translations/

      # Build and push the Docker image to ECR
      - $(aws ecr get-login --no-include-email --region us-east-1)
      - docker build -t translate-web-app .
      - docker tag translate-web-app:latest ${WEB_APP_IMAGE_URI}:latest
      - docker push ${WEB_APP_IMAGE_URI}:latest
  post_build:
    commands:
      - printf '[{"name":"WebAppContainer", "imageUri":"%s:latest"}]' $WEB_APP_IMAGE_URI > imagedefinitions.json

artifacts:
  files:
    - imagedefinitions.json

<p>{{ _('This is a newly added sentence.This should appear correctly translated as soon as the build process completes.Amazon Translate makes this very easy!') }}</p>

[…]
Translating entry 'Settings' to language 'es'
Translating entry 'Get help' to language 'es'
Translating entry 'Welcome to this site.' to language 'es'
Translating entry 'Use the links on the left to navigate through the site, or sign in using the button at the upper right corner.' to language 'es'
Translating entry 'All text shown in this application is automatically translated as part of your Continuous Integration / Continuous Delivery (CI/CD) flow.' to language 'es'
Translating entry 'This is a newly added sentence.This should appear correctly translated as soon as the build process completes.Amazon Translate makes this very easy!' to language 'es'

#!/usr/bin/env python
import argparse
from botocore.exceptions import ClientError
import boto3
import hashlib
import os
import polib
import time

DEFAULT_SOURCE_LANG = 'en'
DEFAULT_TRANSLATIONS_ROOT_DIR = 'app/translations'

def validate_translations_dir(dir):
    """Checks that the given translations directory exists.

    Args:
        dir: The relative path to the directory.

    Raises:
        Exception: The translations directory does not exist.
    """
    if not os.path.exists(dir):
        raise Exception("Translations directory '{}' does not exist".format(dir))

def validate_cache_bucket(s3_client):
    """Checks that the S3 bucket to be used as the cache exists and can be
    accessed with the AWS credentials available to this script.

    Args:
        s3_client: The boto3 S3 client.

    Raises:
        Exception: The TRANSLATIONS_CACHE_BUCKET enviroment variable is not
        set, or the S3 bucket is either unaccessible or does not exist.
    """
    # Check that the cache bucket actually exists in S3.
    bucket = os.environ['TRANSLATIONS_CACHE_BUCKET']
    try:
        s3_client.head_bucket(Bucket=bucket)
    except ClientError as e:
        raise Exception("The translations cache bucket '{}' does not exist or cannot be accessed".format(bucket))

def get_pofile_path_for_language(lang, translations_dir):
    """Returns the relative path to the .PO file for the specified language.

    Args:
        lang: The language code.
        translations_dir: The relative path to the directory containing all
            translations-related files.

    Returns:
        A string with the relative path to the .PO file.
    """
    return "{}/{}/LC_MESSAGES/messages.po".format(translations_dir, lang)

def get_target_languages(translations_dir):
    """Returns all languages that the app should be translated into.

    Args:
        translations_dir: The relative path to the directory containing all
            translations-related files.

    Returns:
        A list of language codes.For example:

        ['es', 'fr']
    """
    target_languages = next(os.walk(translations_dir))[1]
    print('Detected languages:', target_languages)
    return target_languages

def get_hash_from_file(filename):
    """Calculates and returns an SHA256 hash of the specified file.

    Args:
        filename: The path to the file whose hash is to be calculated.

    Returns:
        An SHA256 hash in hexadecimal representation.
    """
    sha256_hash = hashlib.sha256()
    with open(filename, "rb") as f:
        # Read from file in 4KB blocks, this will allow us to handle large files.
        for byte_block in iter(lambda: f.read(4096), b""):
            sha256_hash.update(byte_block)

    return sha256_hash.hexdigest()

def should_cache_be_used(cache_bucket, s3_client):
    """Determines whether the cache represented by the specified S3 bucket
    is up-to-date and should be used in the translation process.

    The cache is considered to be up-to-date if the bucket contains a file with
    an SHA256 hash of the template file, and this hash matches the one of the
    current template file.

    Args:
        cache_bucket: The name of the S3 bucket to be used as the cache.
        s3_client: The boto3 S3 client.

    Returns:
        A tuple containing, in this order, a boolean indicating whether the
        cache is considered to be up-to-date and the current SHA256 hash of the
        messages file.
    """
    # Calculate the SHA256 hash of the original messages file.
    current_hash = get_hash_from_file('messages.pot')

    # Get the hash of the messages file from the cache bucket.
    use_cache = False
    try:
        hash_response = s3_client.get_object(Bucket=cache_bucket, Key='messages.pot.sha256')
        latest_hash = hash_response['Body'].read().decode('utf-8')

        if latest_hash == current_hash:
            print('Hashes match, will try to use cache first')
            use_cache = True
        else:
            print('Hashes do not match, cache will be skipped')

    except ClientError as e:
        print("'messages.pot.sha256' is not present in bucket")

    return (use_cache, current_hash)

def translate_language(src_lang, dest_lang, pofile_path):
    """Translate the app strings from and to the specified languages by
    invoking the Amazon Translate API.

    When this function completes, the translated strings will be written to the
    .PO file of the destination language.

    Args:
        src_lang: The ISO code of the language to translate from, e.g., 'en'.
        dest_lang: The ISO code of the language to translate to, e.g., 'es'.
        pofile_path: The path to the .PO file containing the strings to be
            translated.
    """
    print("Translating from '{}' to '{}'".format(src_lang, dest_lang))

    translate = boto3.client('translate')

    po = polib.pofile(pofile_path)
    for entry in po:
        print("Translating entry '{}' to language '{}'".format(entry.msgid, dest_lang))

        # Calculate the time that the request takes
        time_before = time.time_ns()
        response = translate.translate_text(
            Text=entry.msgid,
            SourceLanguageCode=src_lang,
            TargetLanguageCode=dest_lang
        )
        time_after = time.time_ns()

        entry.msgstr = response['TranslatedText']

        # Wait if needed to avoid throtling
        time_diff = time_after - time_before
        if time_diff < 50000:
            time.sleep(0.05)

    po.save(pofile_path)

def generate_translations_with_cache(src_lang, translations_dir, cache_bucket):
    """Translate the app to all applicable languages, using the given S3 bucket
    name as cache when possible.

    Args:
        src_lang: The ISO code of the language to translate from, e.g., 'en'.
        translations_dir: The relative path to the directory containing all
            translations-related files.
        cache_bucket: The name of the S3 bucket to be used as the cache.
    """
    s3_client = boto3.client('s3')

    # Check that the cache bucket exists and can be accessed.
    validate_cache_bucket(s3_client)

    # Determine whether we should try to get translations from the cache first.
    # This will save us requests to the Amazon Translate API.
    use_cache, current_hash = should_cache_be_used(cache_bucket, s3_client)

    target_languages = get_target_languages(translations_dir)

    for dest_lang in target_languages:
        pofile_path = get_pofile_path_for_language(dest_lang, translations_dir)
        if use_cache:
            try:
                response = s3_client.get_object(Bucket=cache_bucket, Key="{}/messages.po".format(dest_lang))
                with open(pofile_path, "wb") as f:
                    f.write(response['Body'].read())
                print("Retrieved messages file from cache")
            except ClientError as e:
                print("Cache file not found, will regenerate")
                translate_language(src_lang, dest_lang, pofile_path)
        else:
            translate_language(src_lang, dest_lang, pofile_path)

        # Upload localized messages file to S3 bucket.
        print("Uploading to cache")
        s3_client.put_object(
            Bucket=cache_bucket,
            Key="{}/messages.po".format(dest_lang),
            Body=open(pofile_path, "rb")
        )

    # Update hash in cache bucket, if needed.
    if not use_cache:
        print("Uploading hash")
        s3_client.put_object(
            Bucket=cache_bucket,
            Key='messages.pot.sha256',
            Body=current_hash
        )

def generate_translations_without_cache(src_lang, translations_dir):
    """Translate the app to all applicable target languages, without using
    a cache.

    Args:
        src_lang: The ISO code of the language to translate from, e.g., 'en'.
        translations_dir: The relative path to the directory containing all
            translations-related files.
    """
    target_languages = get_target_languages(translations_dir)

    for dest_lang in target_languages:
        pofile_path = get_pofile_path_for_language(dest_lang, translations_dir)
        translate_language(src_lang, dest_lang, pofile_path)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--source-language",
                        help="the ISO code of the source language to translate from (e.g., 'en'), defaults to '{}'".format(DEFAULT_SOURCE_LANG),
                        default=DEFAULT_SOURCE_LANG)
    parser.add_argument("--translations-dir",
                        help="the relative path to the directory containing all translation data, defaults to '{}'".format(DEFAULT_TRANSLATIONS_ROOT_DIR),
                        default=DEFAULT_TRANSLATIONS_ROOT_DIR)
    parser.add_argument("--no-cache",
                        help="disable the translations cache",
                        action="store_true")
    args = parser.parse_args()

    print("Source language is:      {}".format(args.source_language))
    print("Translations dir is:     {}".format(args.translations_dir))

    # Check that the translations folder exists.
    validate_translations_dir(args.translations_dir)

    if args.no_cache:
        generate_translations_without_cache(args.source_language, args.translations_dir)
    else:
        if 'TRANSLATIONS_CACHE_BUCKET' not in os.environ:
            raise Exception("Environment variable TRANSLATIONS_CACHE_BUCKET is not set, please specify a value")

        generate_translations_with_cache(args.source_language,
                                         args.translations_dir,
                                         os.environ['TRANSLATIONS_CACHE_BUCKET'])

$ ./generate_translations.py
Source language is:      en
Translations dir is:     app/translations
'messages.pot.sha256' is not present in bucket
Detected languages: ['es']
Translating from 'en' to 'es'
Translating entry 'Blog' to language 'es'
Translating entry 'Subscription plans' to language 'es'
Translating entry 'My account' to language 'es'
Translating entry 'Log off' to language 'es'

$ ./generate_translations.py
Source language is:      en
Translations dir is:     app/translations
Hashes match, will try to use cache first
Detected languages: ['es']
Retrieved messages file from cache