Python中高效统计两文本间共同词汇出现次数的教程

import requests
import re
from collections import Counter
from bs4 import BeautifulSoup

# 示例：获取两个维基百科页面的内容
url_a = 'https://en.wikipedia.org/wiki/Leonhard_Euler'
url_b = 'https://en.wikipedia.org/wiki/Carl_Friedrich_Gauss'

# 使用requests和BeautifulSoup获取并解析网页文本
try:
    txt_a = BeautifulSoup(requests.get(url_a).content, 'html.parser').get_text()
    txt_b = BeautifulSoup(requests.get(url_b).content, 'html.parser').get_text()
except requests.exceptions.RequestException as e:
    print(f"获取网页内容失败: {e}")
    # 提供备用文本或退出
    txt_a = "Leonhard Euler was a pioneering Swiss mathematician and physicist. He made important discoveries in fields as diverse as infinitesimal calculus and graph theory. He also introduced much of the modern mathematical terminology and notation, particularly for mathematical analysis, such as the notion of a mathematical function. He is also renowned for his work in mechanics, fluid dynamics, optics, astronomy, and music theory. Euler was one of the most eminent mathematicians of the 18th century and is considered one of the greatest mathematicians of all time. He is also widely considered to be the most prolific mathematician ever, with his collected works filling 60–80 quarto volumes. He spent most of his adult life in St. Petersburg, Russia, and in Berlin, Prussia. A statement attributed to Pierre-Simon Laplace expresses Euler's influence on mathematics: Read Euler, read Euler, he is the master of us all."
    txt_b = "Carl Friedrich Gauss was a German mathematician and physicist who made significant contributions to many fields in mathematics and science. Gauss contributed to number theory, algebra, statistics, analysis, differential geometry, geodesy, geophysics, mechanics, electrostatics, astronomy, matrix theory, and optics. Sometimes referred to as the Princeps mathematicorum (Latin for 'the foremost of mathematicians') and 'the greatest mathematician since antiquity', Gauss had an exceptional influence in many fields of mathematics and science and is ranked as one of history's most influential mathematicians. He was a child prodigy. Gauss completed his magnum opus, Disquisitiones Arithmeticae, in 1798 at the age of 21, though it was not published until 1801. This work was fundamental in consolidating number theory as a discipline and has shaped the field to the present day."

# 初步清洗：转换为小写并按非字母数字字符分割
# re.split(r'\W+', text.lower()) 会将所有非字母数字字符作为分隔符，并返回一个词汇列表
words_a = re.split(r'\W+', txt_a.lower())
words_b = re.split(r'\W+', txt_b.lower())

# 移除可能存在的空字符串（例如，文本开头或结尾的非字母数字字符）
words_a = [word for word in words_a if word]
words_b = [word for word in words_b if word]

print(f"文本A词汇数量: {len(words_a)}")
print(f"文本B词汇数量: {len(words_b)}")

# 使用Counter统计每个文本的词频
cnt_a = Counter(words_a)
cnt_b = Counter(words_b)

print("\n文本A中'euler'的出现次数:", cnt_a['euler'])
print("文本B中'gauss'的出现次数:", cnt_b['gauss'])
print("文本A中'the'的出现次数:", cnt_a['the'])
print("文本B中'the'的出现次数:", cnt_b['the'])

# 找出所有共同词汇（键的交集）
common_words_keys = cnt_a.keys() & cnt_b.keys()

# 构建一个字典，存储共同词汇及其在两文本中的频率
# 按照总频率降序排列
all_common_word_counts = {}
for word in common_words_keys:
    all_common_word_counts[word] = (cnt_a[word], cnt_b[word])

# 打印出现频率最高的共同词汇
sorted_common_words = sorted(
    all_common_word_counts.items(),
    key=lambda kv: sum(kv[1]),  # 按两文本总出现次数排序
    reverse=True
)

print("\n出现频率最高的共同词汇及其在两文本中的次数 (前10名):")
for word, counts in sorted_common_words[:10]:
    print(f"'{word}': (文本A: {counts[0]}, 文本B: {counts[1]})")

# 示例输出：
# 'the': (文本A: 596, 文本B: 991)
# 'of': (文本A: 502, 文本B: 874)
# 'in': (文本A: 226, 文本B: 576)
# ...

# 定义一个关键词列表
wordset = {'mathematical', 'equation', 'theorem', 'university', 'integral', 'basel', 'physics'}

print("\n指定关键词在文本A中的出现次数:")
for word in wordset:
    # 使用.get()方法，如果词不存在则返回0
    count = cnt_a.get(word, 0)
    if count > 0:
        print(f"'{word}': {count}")

print("\n指定关键词在文本B中的出现次数:")
for word in wordset:
    count = cnt_b.get(word, 0)
    if count > 0:
        print(f"'{word}': {count}")

# 也可以直接生成一个字典：
keyword_counts_a = {k: cnt_a.get(k, 0) for k in wordset}
keyword_counts_b = {k: cnt_b.get(k, 0) for k in wordset}

print("\n文本A中关键词统计结果:", keyword_counts_a)
print("文本B中关键词统计结果:", keyword_counts_b)