算数ドリルを秒で解くプログラムに Amazon Textract を使って挑戦してみる

from PIL import Image
import boto3
img_file = './Hello_Amazon_Textract.png'
with open(img_file,'rb') as f:
    img_bytes = f.read()
textract = boto3.client('textract',region_name='us-east-1') # 東京リージョンでは Textract が使えないので、バージニア北部を利用
textract.detect_document_text(
    Document={
        'Bytes': img_bytes,
    }
)

{'DocumentMetadata': {'Pages': 1},
 'Blocks': [{'BlockType': 'PAGE', # ページを検出
   # Geometryでページが画像のどこにあるかを指している。この場合は画像全体
   'Geometry': {'BoundingBox': {'Width': 1.0,'Height': 1.0,'Left': 0.0,'Top': 0.0},
    'Polygon': [{'X': 0.0, 'Y': 0.0062385001219809055},
     {'X': 0.9988686442375183, 'Y': 0.0},
     {'X': 1.0, 'Y': 1.0},
     {'X': 0.0011617843993008137, 'Y': 1.0}]},
   'Id': '33487931-8ea7-4150-846c-a136d17ff7e7',
   'Relationships': [{'Type': 'CHILD',
     'Ids': ['184e1993-8497-4f57-9f1f-ce26d484af9f']}]}, # 子要素のID → 下のLINE(行)を指している
  {'BlockType': 'LINE',
   'Confidence': 99.92082214355469,
   'Text': 'Hello Amazon Textract', # Hello Amazon Textract という行を検出
   'Geometry': {'BoundingBox': {'Width': 0.36586683988571167,
     'Height': 0.41139984130859375,
     'Left': 0.6142960786819458,
     'Top': 0.4278397262096405},
    'Polygon': [{'X': 0.6142960786819458, 'Y': 0.5191991925239563},
     {'X': 0.978757917881012, 'Y': 0.4278397262096405},
     {'X': 0.9801629185676575, 'Y': 0.7478801012039185},
     {'X': 0.6157010793685913, 'Y': 0.8392395973205566}]},
   'Id': '184e1993-8497-4f57-9f1f-ce26d484af9f', # ページの子要素から指し示された ID
   'Relationships': [{'Type': 'CHILD',
     # 行の子要素として 3 つのIDを持つ → 下の Word (単語) 3 つを指している
     'Ids': ['ee52f921-e926-44e6-8a6f-3082023797da',
      '8ad19799-7055-4da3-a1ad-6e6f2020e5d4',
      '1695052c-0212-4427-b836-ec8719bf0d6d']}]},
  # 以下、Hello, Amazon, Texract という単語の検出をした結果と場所などを表示
  {'BlockType': 'WORD',
   'Confidence': 99.88762664794922,
   'Text': 'Hello',
   'Geometry': {'BoundingBox': {'Width': 0.08201014250516891,
     'Height': 0.2605866491794586,
     'Left': 0.6142960786819458,
     'Top': 0.4989061951637268},
    'Polygon': [{'X': 0.6142960786819458, 'Y': 0.5191991925239563},
     {'X': 0.6952512860298157, 'Y': 0.4989061951637268},
     {'X': 0.6963062286376953, 'Y': 0.7391998767852783},
     {'X': 0.6153509616851807, 'Y': 0.7594928741455078}]},
   'Id': 'ee52f921-e926-44e6-8a6f-3082023797da'},
  {'BlockType': 'WORD',
   'Confidence': 99.95772552490234,
   'Text': 'Amazon',
   'Geometry': {'BoundingBox': {'Width': 0.13044267892837524,
     'Height': 0.31211456656455994,
     'Left': 0.7037955522537231,
     'Top': 0.46519193053245544},
    'Polygon': [{'X': 0.7037955522537231, 'Y': 0.4975821077823639},
     {'X': 0.8330102562904358, 'Y': 0.46519193053245544},
     {'X': 0.8342382907867432, 'Y': 0.7449163198471069},
     {'X': 0.7050235867500305, 'Y': 0.7773064970970154}]},
   'Id': '8ad19799-7055-4da3-a1ad-6e6f2020e5d4'},
  {'BlockType': 'WORD',
   'Confidence': 99.9171142578125,
   'Text': 'Textract',
   'Geometry': {'BoundingBox': {'Width': 0.1349683552980423,
     'Height': 0.3111570477485657,
     'Left': 0.8451945781707764,
     'Top': 0.4702499806880951},
    'Polygon': [{'X': 0.8451945781707764, 'Y': 0.5037769079208374},
     {'X': 0.9789441227912903, 'Y': 0.4702499806880951},
     {'X': 0.9801629781723022, 'Y': 0.7478801012039185},
     {'X': 0.8464134335517883, 'Y': 0.7814069986343384}]},
   'Id': '1695052c-0212-4427-b836-ec8719bf0d6d'}],
 'DetectDocumentTextModelVersion': '1.0',
 'ResponseMetadata': {'RequestId': '2ed02889-cd52-4a4f-8f19-c022234a2eac',
  'HTTPStatusCode': 200,
  'HTTPHeaders': {'date': 'Fri, 18 Sep 2020 02:35:41 GMT',
   'content-type': 'application/x-amz-json-1.1',
   'content-length': '2424',
   'connection': 'keep-alive',
   'x-amzn-requestid': '2ed02889-cd52-4a4f-8f19-c022234a2eac'},
  'RetryAttempts': 0}}

# 必要なライブラリを読み込む
from PIL import Image, ImageDraw, ImageFont
import boto3, requests, os
from math import ceil
from matplotlib import pyplot
# textract を利用するために boto3 で textract クライアントを生成
# 東京リージョンではまだ利用できないため、バージニア北部リージョンを利用
textract = boto3.client('textract',region_name='us-east-1')

# 問題を格納しているディレクトリ
IMG_DIR = './question/'
# 試してみるファイル
IMG_FILE = '00000.png'
IMG_PATH = IMG_DIR + IMG_FILE

# まずはテキスト検出 API を実行し、その結果を画像に描画する
with open(IMG_PATH,'rb') as f:
    img_bytes = f.read()
response = textract.detect_document_text(
    Document={
        'Bytes': img_bytes,
    }
)
img = Image.open(IMG_PATH).convert('RGB')
d = ImageDraw.Draw(img)
w,h = img.size
font = ImageFont.truetype("DejaVuSansMono.ttf", 24)
for block in response['Blocks']:
    if block['BlockType'] == 'WORD':
        left = int(w*block['Geometry']['BoundingBox']['Left'])
        top = int(h*block['Geometry']['BoundingBox']['Top'])
        right = ceil(w*block['Geometry']['BoundingBox']['Left']) + w*block['Geometry']['BoundingBox']['Width']
        bottom = ceil(h*block['Geometry']['BoundingBox']['Top']) + h*block['Geometry']['BoundingBox']['Height']
        d.rectangle([(left, top), (right, bottom)], outline='lime', width=2)
        d.text((left, top+20), block['Text'], fill='red', align='center',font=font)
# 画像を表示 ( jupyter 環境を想定)
img

img = Image.open(IMG_PATH)
w,h = img.size
questions = []
# 演算子とその位置を取得
for block in response['Blocks']:
    if block['BlockType'] == 'WORD':
        if block['Text'] in ['-','+']: # - か + を取得
            left = int(w*block['Geometry']['BoundingBox']['Left'])
            top = int(h*block['Geometry']['BoundingBox']['Top'])
            right = ceil(w*block['Geometry']['BoundingBox']['Left'] + w*block['Geometry']['BoundingBox']['Width'])
            bottom = ceil(h*block['Geometry']['BoundingBox']['Top'] + h*block['Geometry']['BoundingBox']['Height'])
            center = ((left+right)//2,(top+bottom)//2)
            operator_symbol = block['Text']
            questions.append({
                'operator_symbol':{
                    'left': left,
                    'top': top,
                    'right': right,
                    'bottom': bottom,
                    'center': center,
                    'operator_symbol': operator_symbol
                }
            })

# 結果確認
for question in questions:
    print(question)

{'operator_symbol': {'left': 99, 'top': 99, 'right': 119, 'bottom': 119, 'center': (109, 109), 'operator_symbol': '+'}}
{'operator_symbol': {'left': 499, 'top': 99, 'right': 519, 'bottom': 120, 'center': (509, 109), 'operator_symbol': '+'}}
{'operator_symbol': {'left': 103, 'top': 347, 'right': 115, 'bottom': 353, 'center': (109, 350), 'operator_symbol': '-'}}
{'operator_symbol': {'left': 499, 'top': 339, 'right': 519, 'bottom': 360, 'center': (509, 349), 'operator_symbol': '+'}}
{'operator_symbol': {'left': 103, 'top': 587, 'right': 115, 'bottom': 593, 'center': (109, 590), 'operator_symbol': '-'}}
{'operator_symbol': {'left': 499, 'top': 579, 'right': 519, 'bottom': 599, 'center': (509, 589), 'operator_symbol': '+'}}
{'operator_symbol': {'left': 99, 'top': 819, 'right': 119, 'bottom': 839, 'center': (109, 829), 'operator_symbol': '+'}}
{'operator_symbol': {'left': 499, 'top': 819, 'right': 519, 'bottom': 839, 'center': (509, 829), 'operator_symbol': '+'}}
{'operator_symbol': {'left': 103, 'top': 1067, 'right': 115, 'bottom': 1073, 'center': (109, 1070), 'operator_symbol': '-'}}
{'operator_symbol': {'left': 499, 'top': 1059, 'right': 518, 'bottom': 1080, 'center': (508, 1069), 'operator_symbol': '+'}}

# 検出した数値のみのリスト
extract_numbers = []
for block in response['Blocks']:
    if block['BlockType'] == 'WORD' and block['Text'].isnumeric():
        number = block['Text']
        left = int(w*block['Geometry']['BoundingBox']['Left'])
        top = int(h*block['Geometry']['BoundingBox']['Top'])
        right = ceil(w*block['Geometry']['BoundingBox']['Left'] + w*block['Geometry']['BoundingBox']['Width'])
        bottom = ceil(h*block['Geometry']['BoundingBox']['Top'] + h*block['Geometry']['BoundingBox']['Height'])
        center = ((left+right)//2,(top+bottom)//2)
        extract_numbers.append({
                'left': left,
                'top': top,
                'right': right,
                'bottom': bottom,
                'center': center,
                'number': number
        })

# 演算子と ± 20 px にある高さの、一番近い数字を取得（加算減算する数字）
margin = 20
diff = 99999 # 演算子と数値の横方向の距離を大きい数字で初期化、小さければその値を採用する
for i,question in enumerate(questions):
    diff = 99999
    candidates = []
    for extract_number in extract_numbers:
        if question['operator_symbol']['center'][1] - margin <= extract_number['center'][1] <= question['operator_symbol']['center'][1] + margin:
            candidates.append(extract_number)
    for candidate in candidates:
        if diff > candidate['center'][0] - question['operator_symbol']['center'][0] > 0:
            diff = question['operator_symbol']['center'][0] - candidate['center'][0]
            questions[i]['y'] = candidate
# y キー を持たない（= 計算問題ではない演算子）を除去する
for question in questions:
    if 'y' in question:
        pass
    else:
        questions.remove(question)

# 加算減算する数値として使用したものをextract_numbersから除外
for question in questions:
    if question['y'] in extract_numbers:
        extract_numbers.remove(question['y'])

# 結果確認
for question in questions:
    formula = 'X' + ' ' + question['operator_symbol']['operator_symbol'] + ' ' + question['y']['number']
    print(formula)

X + 1915
X + 9540
X - 1376
X + 286
X - 260
X + 2434
X + 1875
X + 7975
X - 1058
X + 9855

margin = 20
diff = 99999 # 演算子と数値の横方向の距離を大きい数字で初期化、小さければその値を採用する
for i,question in enumerate(questions):
    diff = 99999
    candidates = []
    for extract_number in extract_numbers:
        # 筆算において数字は右詰めされるため、右端の px 値で評価する
        if question['y']['right'] - margin <= extract_number['right'] <= question['y']['right'] + margin:
            candidates.append(extract_number)
    for candidate in candidates:
        if diff > question['y']['center'][1] - candidate['center'][1] > 0:
            diff = question['y']['center'][1] - candidate['center'][1]
            questions[i]['x'] = candidate
# 結果確認
for question in questions:
    formula = question['x']['number'] + ' ' + question['operator_symbol']['operator_symbol'] + ' ' + question['y']['number']
    print(formula)

7221 + 1915
1486 + 9540
1613 - 1376
3880 + 286
5671 - 260
7595 + 2434
3000 + 1875
8243 + 7975
8866 - 1058
1124 + 9855

# eval メソッドを使って計算を解く
for i,question in enumerate(questions):
    formula = str(question['x']['number']) + ' ' + question['operator_symbol']['operator_symbol'] + ' ' + question['y']['number']
    questions[i]['formula'] = formula
    questions[i]['answer'] = {'number':eval(formula)}
# 結果確認
for question in questions:
    print(question['formula'] + ' = ' + str(question['answer']['number']))

7221 + 1915 = 9136
1486 + 9540 = 11026
1613 - 1376 = 237
3880 + 286 = 4166
5671 - 260 = 5411
7595 + 2434 = 10029
3000 + 1875 = 4875
8243 + 7975 = 16218
8866 - 1058 = 7808
1124 + 9855 = 10979

img = Image.open(IMG_PATH)
d = ImageDraw.Draw(img)
for i,question in enumerate(questions):
    questions[i]['answer']['top'] = question['y']['top'] + (question['y']['top'] - question['x']['top'])

    # 文字数によって開始位置を変更する
    # 1 文字あたりどれくらいの幅を取るかを width_unit に格納し、その幅分で文字の左端位置を調整する
    width_unit = ((question['y']['right'] - question['y']['left']) / len(str(question['y']['number'])))
    questions[i]['answer']['left'] = int(question['y']['left'] - (len(str(question['answer']['number'])) - len(str(question['y']['number'])))*width_unit)

    font = ImageFont.truetype("DejaVuSansMono.ttf", round(round(width_unit)*(10/6))) # 10/6 はフォントと幅を変換する定数
    d.text((questions[i]['answer']['left'], questions[i]['answer']['top']), str(questions[i]['answer']['number']), font=font,fill='black',align='right')
img

$ python main.py ./question/ ./answer/ 20

exec_sec: 15.99182677268982