python如何对长篇文章进行语义分块？

在Python中对长篇文章进行语义分块是一个涉及自然语言处理（NLP）技术的任务，以下是分步实现的详细方法和代码示例：

语义分块的目标是将文本划分为 语义连贯的段落，而非简单的固定长度切割。主要方法分为两类：

import spacy

def semantic_chunking_rule(text, max_chunk_size=500):
    nlp = spacy.load("zh_core_web_sm")  # 中文模型
    doc = nlp(text)

    chunks = []
    current_chunk = []
    current_length = 0

    for sent in doc.sents:
        sent_length = len(sent.text)
        if current_length + sent_length <= max_chunk_size:
            current_chunk.append(sent.text)
            current_length += sent_length
        else:
            chunks.append(" ".join(current_chunk))
            current_chunk = [sent.text]
            current_length = sent_length

    if current_chunk:
        chunks.append(" ".join(current_chunk))

    return chunks

# 使用示例
text = "长篇文章内容..."
chunks = semantic_chunking_rule(text)

from collections import defaultdict

def keyword_based_chunking(text, keywords=["然而", "总之", "综上所述"]):
    chunks = []
    buffer = []

    for paragraph in text.split("\n"):
        buffer.append(paragraph)
        if any(keyword in paragraph for keyword in keywords):
            chunks.append("\n".join(buffer))
            buffer = []

    if buffer:
        chunks.append("\n".join(buffer))

    return chunks

from sentence_transformers import SentenceTransformer
import numpy as np

model = SentenceTransformer('paraphrase-multilingual-mpnet-base-v2')

def semantic_split(te...

点击查看剩余70%