使用Python和NLTK从文本中高效提取名词的教程

# 安装NLTK库
pip install nltk

import nltk

# 下载punkt分词器模型（用于分句）
nltk.download('punkt')
# 下载averaged_perceptron_tagger（用于词性标注）
nltk.download('averaged_perceptron_tagger')
# 下载stopwords（可选，用于移除停用词）
nltk.download('stopwords')

from nltk.tokenize import sent_tokenize

text = "Marriage is a big step in one's life. It involves commitment and shared responsibilities. Python is a powerful programming language."
sentences = sent_tokenize(text)
print("分句结果:", sentences)
# 输出: ['Marriage is a big step in one\'s life.', 'It involves commitment and shared responsibilities.', 'Python is a powerful programming language.']

from nltk.tokenize import word_tokenize

# 以第一个句子为例
first_sentence = sentences[0]
words = word_tokenize(first_sentence)
print("分词结果:", words)
# 输出: ['Marriage', 'is', 'a', 'big', 'step', 'in', 'one', "'s", 'life', '.']

from nltk.corpus import stopwords

stop_words = set(stopwords.words('english'))
filtered_words = [word for word in words if word.lower() not in stop_words]
print("移除停用词后:", filtered_words)
# 输出: ['Marriage', 'big', 'step', 'one', "'s", 'life', '.']

import nltk

tagged_words = nltk.pos_tag(filtered_words)
print("词性标注结果:", tagged_words)
# 输出: [('Marriage', 'NN'), ('big', 'JJ'), ('step', 'NN'), ('one', 'CD'), ("'s", 'POS'), ('life', 'NN'), ('.', '.')]

nouns = [word for word, tag in tagged_words if tag.startswith('NN')]
print("提取的名词:", nouns)
# 输出: ['Marriage', 'step', 'life']

import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords

# 确保已下载NLTK数据
# nltk.download('punkt')
# nltk.download('averaged_perceptron_tagger')
# nltk.download('stopwords')

def extract_nouns(text):
    """
    从给定的文本中提取所有名词。

    Args:
        text (str): 待处理的输入文本。

    Returns:
        list: 包含所有提取到的名词的列表。
    """
    all_nouns = []
    stop_words = set(stopwords.words('english'))

    # 1. 分句
    sentences = sent_tokenize(text)

    for sentence in sentences:
        # 2. 分词
        words = word_tokenize(sentence)

        # 3. 移除停用词（可选，但通常有助于聚焦核心名词）
        # 过滤掉非字母的词语，并转换为小写后与停用词比较
        filtered_words = [word for word in words if word.isalpha() and word.lower() not in stop_words]

        if not filtered_words: # 避免空列表传入pos_tag
            continue

        # 4. 词性标注
        tagged_words = nltk.pos_tag(filtered_words)

        # 5. 提取名词 (标签以'NN'开头)
        nouns_in_sentence = [word for word, tag in tagged_words if tag.startswith('NN')]
        all_nouns.extend(nouns_in_sentence)

    return list(set(all_nouns)) # 使用set去重并转换回列表

# 示例文本，可以替换为LLM的响应
llm_response_text = """
The quick brown fox jumps over the lazy dog. Artificial intelligence is transforming various industries. 
LangChain provides powerful tools for building LLM applications. Understanding context is crucial for natural language processing.
"""

extracted_nouns = extract_nouns(llm_response_text)
print(f"从LLM响应中提取的名词: {extracted_nouns}")

# 另一个示例
another_text = "I have a task that involves extracting nouns from a variable called message: response. I want to display the extracted nouns in the console or print them on the screen. How can I accomplish this task using Python? I have tried using some libraries like NLTK and TextBlob, but I am not sure how to use them correctly. I have also asked GitHub Copilot for help, but it did not generate any useful code. It just showed me some random output that did not work. Can anyone please help me with this problem?"
extracted_nouns_2 = extract_nouns(another_text)
print(f"从另一个文本中提取的名词: {extracted_nouns_2}")

从LLM响应中提取的名词: ['dog', 'tools', 'context', 'intelligence', 'applications', 'fox', 'industries', 'processing', 'LangChain']
从另一个文本中提取的名词: ['task', 'screen', 'help', 'output', 'libraries', 'problem', 'console', 'message', 'response', 'code']

# 假设这是您的LangChain响应
langchain_response = {
    "answer": "The capital of France is Paris. Paris is also known as the City of Light and is a major European city.",
    "sources": ["Wikipedia"]
}

# 提取LLM的回答文本
response_text = langchain_response.get("answer", "")

if response_text:
    extracted_nouns_from_llm = extract_nouns(response_text)
    print(f"从LangChain响应中提取的名词: {extracted_nouns_from_llm}")
else:
    print("LangChain响应中未找到可供提取名词的文本。")