python如何爬取搜索引擎的搜索数据？

可以使用selenium爬取搜索引擎的搜索结果，我们以百度为例，这是运行的结果

#!/usr/local/python3/bin/python3
# -*- coding: utf-8 -*
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

chromeOptions = webdriver.ChromeOptions()
chromeOptions.add_argument("--headless")
chromeOptions.add_argument("--remote-debugging-port=9222")
chromeOptions.add_argument('--no-sandbox')
browser = webdriver.Chrome('/usr/bin/chromedriver',chrome_options=chromeOptions)

browser.get("https://www.baidu.com/")               #进入相关网站
#保存网站截屏图片   


browser.find_element_by_id('kw').send_keys('ai发展前景 ',Keys.RETURN)  # 输入框
browser.find_element_by_id('su').click()  # 点击搜索按钮


try:
    sleep(1)
    # 等待<h3 class="c-title t t tts-title">标签出现
    # element = WebDriverWait(browser, 10).until(
    #     EC.presence_of_element_located((By.CSS_SELECTOR, "h3.c-title.t.tts-title"))
    # )
    # 获取网页内容
    html_content = browser.page_source
    
    # 使用BeautifulSoup解析网页内容
    soup = BeautifulSoup(html_content, 'html.parser')
    
    # 查找所有的<h3>标签
    h3_tags = soup.find_all('h3', class_='c-title t t tts-title')
    
    # 提取<a>链接和标题
    for h3 in h3_tags:
        a_tag = h3.find('a')
        if a_tag:
            link = a_tag['href']
            title = a_tag.get_text(strip=True)
            print(f'Link: {link}')
            print(f'Title: {title}')
            print('-' * 50)
    
    # 关闭WebDriver
    browser.quit()
    
    
finally:
    # 关闭浏览器
    browser.quit()

        
		

点击查看全文