テキスト解析 AI サービス Amazon Comprehend で本を読まずに読書感想文に挑戦してみる

import requests
from html.parser import HTMLParser

class MyHTMLParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.text = ""
        self.extract = False
        
    def handle_starttag(self, tag, attrs):
        if tag == "div":
            self.extract = True
        elif tag in ["rp","rt"]:
            self.extract = False

    def handle_endtag(self, tag):
        if tag in ["rp","rt"]:
            self.extract = True
        elif tag == "div":
            self.extract = False

    def handle_data(self, data):
        if self.extract:
            self.text += data
URL = "https://www.aozora.gr.jp/cards/000035/files/1567_14913.html"
response = requests.get(URL)
response.encoding = response.apparent_encoding
response.text
parser = MyHTMLParser()
parser.feed(response.text)
print(parser.text)

text = parser.text.replace("\r","").replace("\n","").replace("\u3000","").replace("（古伝説と、シルレルの詩から。）底本：「太宰治全集3」ちくま文庫、筑摩書房1988（昭和63）年10月25日初版発行1998（平成10）年6月15日第2刷底本の親本：「筑摩全集類聚版太宰治全集」筑摩書房1975（昭和50）年6月〜1976（昭和51）年6月入力：金川一之校正：高橋美奈子2000年12月4日公開2011年1月17日修正青空文庫作成ファイル：このファイルは、インターネットの図書館、青空文庫（http://www.aozora.gr.jp/）で作られました。入力、校正、制作にあたったのは、ボランティアの皆さんです。●表記について    このファイルは W3C 勧告 XHTML1.1 にそった形式で作成されています。●図書カード","").replace("走れメロス太宰治","")
print(text)

sentences = text.replace("「","").replace("」","").split("。")[:-1]
print(sentences)

import boto3
comprehend_client = boto3.client("comprehend")

from collections import defaultdict
characters = defaultdict(int)
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    for entity in entities_response["Entities"]:
        if entity["Type"] == "PERSON":
            characters[entity["Text"]] += 1
print(sorted(characters.items(), key=lambda x:x[1],reverse=True))

melos_key_phrases = []
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    for entity in entities_response["Entities"]:
        if entity["Type"] == "PERSON":
            if entity["Text"] == "メロス":
                key_phrases_response = comprehend_client.detect_key_phrases(Text=sentence,LanguageCode="ja")
                for key_phrase in key_phrases_response["KeyPhrases"]:
                    if key_phrase["Text"] != "メロス":
                        melos_key_phrases.append(key_phrase["Text"])
print(melos_key_phrases)

melos_positive = []
melos_negative = []
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    for entity in entities_response["Entities"]:
        if entity["Type"] == "PERSON":
            if entity["Text"] == "メロス":
                melos_sentiment_response = comprehend_client.detect_sentiment(Text=sentence,LanguageCode="ja")
                melos_positive.append(melos_sentiment_response["SentimentScore"]["Positive"])
                melos_negative.append(melos_sentiment_response["SentimentScore"]["Negative"])
from matplotlib import pyplot as plt
fig = plt.figure(figsize=(16.0, 10.0))
plt.plot(melos_positive,color="green")
plt.plot(melos_negative,color="red")

selinuntius_key_phrases = []
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    for entity in entities_response["Entities"]:
        if entity["Type"] == "PERSON":
            if entity["Text"] == "セリヌンティウス":
                key_phrases_response = comprehend_client.detect_key_phrases(Text=sentence,LanguageCode="ja")
                for key_phrase in key_phrases_response["KeyPhrases"]:
                    if key_phrase["Text"] != "セリヌンティウス":
                        selinuntius_key_phrases.append(key_phrase["Text"])
print(selinuntius_key_phrases)

selinuntius_positive = []
selinuntius_negative = []
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    for entity in entities_response["Entities"]:
        if entity["Type"] == "PERSON":
            if entity["Text"] == "セリヌンティウス":
                selinuntius_sentiment_response = comprehend_client.detect_sentiment(Text=sentence,LanguageCode="ja")
                selinuntius_positive.append(selinuntius_sentiment_response["SentimentScore"]["Positive"])
                selinuntius_negative.append(selinuntius_sentiment_response["SentimentScore"]["Negative"])
from matplotlib import pyplot as plt
fig = plt.figure(figsize=(16.0, 10.0))
plt.plot(selinuntius_positive,color="green")
plt.plot(selinuntius_negative,color="red")

king_key_phrases = []
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    for entity in entities_response["Entities"]:
        if entity["Type"] == "PERSON":
            if entity["Text"] == "王":
                key_phrases_response = comprehend_client.detect_key_phrases(Text=sentence,LanguageCode="ja")
                for key_phrase in key_phrases_response["KeyPhrases"]:
                    if key_phrase["Text"] != "王":
                        king_key_phrases.append(key_phrase["Text"])
print(king_key_phrases)

king_positive = []
king_negative = []
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    for entity in entities_response["Entities"]:
        if entity["Type"] == "PERSON":
            if entity["Text"] == "王":
                king_sentiment_response = comprehend_client.detect_sentiment(Text=sentence,LanguageCode="ja")
                king_positive.append(king_sentiment_response["SentimentScore"]["Positive"])
                king_negative.append(king_sentiment_response["SentimentScore"]["Negative"])
from matplotlib import pyplot as plt
fig = plt.figure(figsize=(16.0, 10.0))
plt.plot(king_positive,color="green", label="positive")
plt.plot(king_negative,color="red", label="negative")
plt.legend(bbox_to_anchor=(1, 1), loc='upper right', borderaxespad=0, fontsize=18)

key_phrases = []
positives = {"メロス":[],"セリヌンティウス":[],"王":[]}
negatives = {"メロス":[],"セリヌンティウス":[],"王":[]}
entity_include_sentence_idx = []
for i,sentence in enumerate(sentences):
    entities_response = comprehend_client.detect_entities(Text=sentence,LanguageCode="ja")
    entities = []
    in_use = False
    for entity in entities_response["Entities"]:
        if entity["Text"] in ["メロス","セリヌンティウス","王"]:
            in_use = True
            entities.append(entity["Text"])
        if len(entities) >= 1:
            sentiment_response = comprehend_client.detect_sentiment(Text=sentence,LanguageCode="ja")
            for person in ["メロス","セリヌンティウス","王"]:
                if person in entities:
                    positives[person].append(sentiment_response["SentimentScore"]["Positive"])
                    negatives[person].append(sentiment_response["SentimentScore"]["Negative"])
                else:
                    # エンティティとして登場しない場合の処理
                    if positives[person] == []:
                        # 最初は 0
                        positives[person].append(0)
                        negatives[person].append(0)
                    else:
                        # 直前の感情で補完
                        positives[person].append(positives[person][-1])
                        negatives[person].append(negatives[person][-1])
    if in_use:
        entity_include_sentence_idx.append(i)
key_phrases = []
for i in entity_include_sentence_idx:
    key_phrases_response = comprehend_client.detect_key_phrases(Text=sentences[i],LanguageCode="ja")
    temp_key_phrase = ""
    for key_phrase in key_phrases_response["KeyPhrases"]:
        temp_key_phrase = temp_key_phrase + "\n" + key_phrase["Text"]
    key_phrases.append(temp_key_phrase)
import matplotlib as mpl

mpl.rcParams['font.family']='IPAexGothic'
fig = plt.figure(figsize=(16.0, 10.0))
plt.plot(positives["メロス"],color="green", label="メロスP")
plt.plot(negatives["メロス"],color="red", label="メロスN")
plt.plot(positives["セリヌンティウス"],color="blue", label="セリヌンティウスP")
plt.plot(negatives["セリヌンティウス"],color="orange", label="セリヌンティウスN")
plt.plot(positives["王"],color="purple", label="王P")
plt.plot(negatives["王"],color="yellow", label="王N")
plt.legend(bbox_to_anchor=(1, 1), loc='upper right', borderaxespad=0, fontsize=18)
for i,key_phrase in enumerate(key_phrases):
    plt.text(i,1,key_phrase,size=10)

th = 0.5
th_over_positives = {"メロス":[],"セリヌンティウス":[],"王":[]}
th_over_negatives = {"メロス":[],"セリヌンティウス":[],"王":[]}
th_over_key_phrases = []
for i in range(len(key_phrases)):
    if positives["メロス"][i] >= 0.5 or positives["セリヌンティウス"][i] >= 0.5 or positives["王"][i] >= 0.5 or negatives["メロス"][i] >= 0.5 or negatives["セリヌンティウス"][i] >= 0.5 or negatives["王"][i] >= 0.5:
        th_over_positives["メロス"].append(positives["メロス"][i])
        th_over_positives["セリヌンティウス"].append(positives["セリヌンティウス"][i])
        th_over_positives["王"].append(positives["王"][i])
        th_over_negatives["メロス"].append(negatives["メロス"][i])
        th_over_negatives["セリヌンティウス"].append(negatives["セリヌンティウス"][i])
        th_over_negatives["王"].append(negatives["王"][i])
        th_over_key_phrases.append(key_phrases[i])
mpl.rcParams['font.family']='IPAexGothic'
fig = plt.figure(figsize=(16.0, 10.0))
plt.plot(th_over_positives["メロス"],color="green", label="メロスP")
plt.plot(th_over_negatives["メロス"],color="red", label="メロスN")
plt.plot(th_over_positives["セリヌンティウス"],color="blue", label="セリヌンティウスP")
plt.plot(th_over_negatives["セリヌンティウス"],color="orange", label="セリヌンティウスN")
plt.plot(th_over_positives["王"],color="purple", label="王P")
plt.plot(th_over_negatives["王"],color="yellow", label="王N")
plt.legend(bbox_to_anchor=(1, 1), loc='upper right', borderaxespad=0, fontsize=18)
for i,key_phrase in enumerate(th_over_key_phrases):
    height = (i%5) * (1/4)
    plt.text(i,height,key_phrase,size=10)