【言語処理100本ノック解いてみた】 第4章：形態素解析 No.30~No.34

!wget https://nlp100.github.io/data/neko.txt
!apt install mecab libmecab-dev mecab-ipadic-utf8
!mecab neko.txt -o neko.txt.mecab
#作成したファイルの確認
!head -10 neko.txt.mecab

#出力
一	名詞,数,*,*,*,*,一,イチ,イチ
    記号,一般,*,*,*,*,*
EOS
	  記号,一般,*,*,*,*,*
EOS
　	記号,空白,*,*,*,*,　,　,　
吾輩	名詞,代名詞,一般,*,*,*,吾輩,ワガハイ,ワガハイ
は	助詞,係助詞,*,*,*,*,は,ハ,ワ
猫	名詞,一般,*,*,*,*,猫,ネコ,ネコ
で	助動詞,*,*,*,特殊・ダ,連用形,だ,デ,デ

filename = 'neko.txt.mecab'
with open(filename, mode='r', encoding='utf-8') as f:
    sentence_block_list = f.read().split("EOS\n")

sentence_blocks = []
for sentence_block in sentence_block_list:
    morpheme_list = []
    block = sentence_block.split("\n")
    for m in block:
        if m == "":
            continue
        surface, attr = m.split("\t")
        if surface == "":
            continue
        attr = attr.split(",")
        morpheme ={
            "surface": surface,
            "base": attr[6],
            "pos": attr[0],
            "pos1": attr[1],
        }
        morpheme_list.append(morpheme)
    if len(morpheme_list) > 0:
        sentence_blocks.append(morpheme_list)
print(sentence_blocks[:5])

#出力(一部編集)
[[{'surface': '一', 'base': '一', 'pos': '名詞', 'pos1': '数'}], 
[{'surface': '\u3000', 'base': '\u3000', 'pos': '記号', 'pos1': '空白'},
 {'surface': '吾輩', 'base': '吾輩', 'pos': '名詞', 'pos1': '代名詞'},
 {'surface': 'は', 'base': 'は', 'pos': '助詞', 'pos1': '係助詞'},
 {'surface': '猫', 'base': '猫', 'pos': '名詞', 'pos1': '一般'},
 {'surface': 'で', 'base': 'だ', 'pos': '助動詞', 'pos1': '*'},
 {'surface': 'ある', 'base': 'ある', 'pos': '助動詞', 'pos1': '*'},
 {'surface': '。', 'base': '。', 'pos': '記号', 'pos1': '句点'}], ・・・
]]

verb_list = []
for sentence_block in sentence_blocks:
    verb = [block["surface"] for block in sentence_block if block["pos"]=="動詞"]
    verb_list += verb
verb_surface_list = list(set(verb_list))
print(len(verb_surface_list))
print(verb_surface_list[:5])

#出力
3893
['禁じ', 'ぶんなぐる', 'たたく', '出会わ', '枉げ']

verb_list = []
for sentence_block in sentence_blocks:
    verb = [block["base"] for block in sentence_block if block["pos"]=="動詞"]
    verb_list += verb
verb_base_list = list(set(verb_list))
print(len(verb_base_list))
print(verb_base_list[:5])

#出力
2300
['ぶんなぐる', '攫む', '飛ぶ', '集める', '乗り出す']

noun_phrase = []
for sentence_block in sentence_blocks:
    for i in range(len(sentence_block)-2):
        if sentence_block[i]["pos"] == "名詞" and sentence_block[i+1]["surface"] =="の" and sentence_block[i+2]["pos"]=="名詞":
            phrase = sentence_block[i]["surface"] + sentence_block[i+1]["surface"] + sentence_block[i+2]["surface"]
            noun_phrase.append(phrase)
            
noun_phrase = list(set(noun_phrase))
print(len(noun_phrase))
print(noun_phrase[:5])

4924
['車屋の黒', '吾輩の横腹', 'そこの風習', '朝飯の催促', 'こっちの趣向']

noun_phrase = []
for sentence_block in sentence_blocks:
    phrase = ""
    count = 0
    for block in sentence_block:
        if block["pos"] == "名詞":
            phrase += block["surface"]    
            count +=1
        else:
            if count >= 2:
                noun_phrase.append(phrase)
                phrase = ""
                count = 0
            else:
                phrase = ""
                count = 0
                     
noun_phrase = list(set(noun_phrase))
print(len(noun_phrase))
print(noun_phrase[:5])

4454
['暑中休暇後', '百騎町', '明治十一年九月廿八日', 'たまえ月十円', '書生同様取次']