主页 / 问题 / 793660
Accepted
MaxU - stop genocide of UA
Asked:2020-03-05 08:05:02 +0000 UTC2020-03-05 08:05:02 +0000 UTC

文本中单词频率特征的可视化

  • 772

在对自然文本进行分类时，出现了一个问题，即如何在有和没有文本规范化的情况下直观地显示文本的频率特征。

首先想到的是条形图，但不可能超过 20-30 个单词——一切都太小而且难以理解。

通过规范化，我的意思是将词带入它们的正常（规范）形式。

我尝试以答案的形式找到解决方案，但看看其他解决方案和想法会很有趣。

python

1 个回答

  1. Best Answer

    在使用NLTK和pymorphy2对文本进行标记和规范化之后，我决定使用WordCloud (c) Andreas Mueller。

    示例（下面的代码）：

    安东·契诃夫。全集和信件 - ModernLib.Ru.txt： 在此处输入图像描述

    普希金亚历山大。完整的诗集-royallib.ru.txt： 在此处输入图像描述

    编码：

    import os
import requests
from operator import attrgetter
from pathlib import Path
#import pandas as pd
import nltk
from nltk import sent_tokenize, word_tokenize, regexp_tokenize
from nltk.corpus import stopwords
import pymorphy2
from wordcloud import WordCloud
import matplotlib.pyplot as plt


# https://raw.githubusercontent.com/stopwords-iso/stopwords-ru/master/stopwords-ru.txt
def read_stopwords(path='./stopwords-ru.txt', encoding='utf-8'):
    stopwords_en = stopwords.words('english')
    with open(path, encoding=encoding) as f:
        stopwords_ru = f.read().split('\n')
    return set(stopwords_ru) | set(stopwords_en)


def normalize_tokens(tokens):
    morph = pymorphy2.MorphAnalyzer()
    return [morph.parse(tok)[0].normal_form for tok in tokens]


def remove_stopwords(tokens, stopwords=None, min_length=4):
    if not stopwords:
        return tokens
    stopwords = set(stopwords)
    tokens = [tok
              for tok in tokens
              if tok not in stopwords and len(tok) >= min_length]
    return tokens


def plot_word_cloud(text, picture_fn='out.png', stopwords=None,
                    normalize=True, regexp=r'(?u)\b\w{4,}\b', **wc_kwargs):
    words = [w for sent in sent_tokenize(text)
             for w in regexp_tokenize(sent, regexp)]
    if normalize:
        words = normalize_tokens(words)
    if stopwords:
        words = remove_stopwords(words, stopwords)
    wc = WordCloud(**wc_kwargs).generate(' '.join(words))
    plt.figure(figsize=(12,10))
    plt.imshow(wc, interpolation="bilinear")
    plt.axis("off")
    plt.savefig(picture_fn)

def get_text(url, encoding='utf-8', to_lower=True):
    url = str(url)
    if url.startswith('http'):
        r = requests.get(url)
        if not r.ok:
            r.raise_for_status()
        return r.text.lower() if to_lower else r.text
    elif os.path.exists(url):
        with open(url, encoding=encoding) as f:
            return f.read().lower() if to_lower else f.read()
    else:
        raise Exception('parameter [url] can be either URL or a filename')


stopwords_ru = read_stopwords('./stopwords-ru.txt')

# Понедельник начинается в субботу
url='https://www.e-reading.club/txt.php/55060/%D0%A1%D1%82%D1%80%D1%83%D0%B3%D0%B0%D1%86%D0%BA%D0%B8%D0%B9_-_%D0%9F%D0%BE%D0%BD%D0%B5%D0%B4%D0%B5%D0%BB%D1%8C%D0%BD%D0%B8%D0%BA_%D0%BD%D0%B0%D1%87%D0%B8%D0%BD%D0%B0%D0%B5%D1%82%D1%81%D1%8F_%D0%B2_%D1%81%D1%83%D0%B1%D0%B1%D0%BE%D1%82%D1%83.txt'
text = get_text(url)
plot_word_cloud(text, 'ponedelnik_norm.png', stopwords=stopwords_ru, max_words=100,
                background_color='black', normalize=True)

# Пушкин Александр. Полное собрание стихотворений - royallib.ru.txt
url='./Пушкин Александр. Полное собрание стихотворений - royallib.ru.txt'
text = get_text(url, encoding='cp1251')
plot_word_cloud(text, 'pushkin_norm.png', stopwords=stopwords_ru, max_words=100,
                background_color='black', normalize=True)

# Чехов Антон. Полное собрание сочинений и писем - ModernLib.Ru.txt
url = './Чехов Антон. Полное собрание сочинений и писем - ModernLib.Ru.txt'
text = get_text(url, encoding='cp1251')
plot_word_cloud(text, 'chekhov_norm.png', stopwords=stopwords_ru, max_words=100,
                background_color='black', normalize=True)
    • 7

相关问题