1.反webdriver自动化检测爬取
有的网站会有检测webdriver的反爬手段，这时候就需要做一些操作防止被网页检测到，下面直接给出固定代码段方法：
from selenium.webdriver.support.ui import WebDriverWait# 配置 ChromeOptions 防止被检测
options = webdriver.ChromeOptions()
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--incognito") # 使用隐身模式
# options.add_argument('--headless') # 无头浏览，开发和调试时可以注释掉这行# 创建 WebDriver 实例
driver = webdriver.Chrome(options=options)
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": """Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"""
})
wait = WebDriverWait(driver, 10)网站实战反反爬：
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support.ui import WebDriverWait# 配置 ChromeOptions 防止被检测
options = webdriver.ChromeOptions()
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--incognito") # 使用隐身模式
# options.add_argument('--headless') # 无头浏览，开发和调试时可以注释掉这行# 创建 WebDriver 实例
driver = webdriver.Chrome(options=options)
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": """Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"""
})
wait = WebDriverWait(driver, 10)
driver.get('https://antispider1.scrape.center/')
time.sleep(5)
all_data_dict = {}
for i in range(10):titles = driver.find_elements(By.XPATH, '//*[@id="index"]/div[1]/div[1]/div/div/div/div[2]/a/h2')grades = driver.find_elements(By.XPATH, '//*[@id="index"]/div[1]/div[1]/div/div/div/div[3]/p[1]')data_dict = {}for title, grade in zip(titles, grades):data_dict[title.text] = grade.textall_data_dict.update(data_dict)print(f"页数 {i + 1}: {data_dict}")try:search_next = driver.find_element(By.XPATH, '//*[@id="index"]/div[2]/div/div/div/button[2]/i').click()time.sleep(3)except Exception as e:print(f"单击下一页出错：{e}")# 打印所有页面的数据
print('-' * 80)
print("所有数据：", all_data_dict)
time.sleep(3)
driver.quit()
2.boss直聘多页爬取
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import timedriver = webdriver.Chrome()
driver.get('https://www.zhipin.com/?ka=header-home-logo')
time.sleep(3)searching = driver.find_element(By.XPATH, '//*[@id="wrap"]/div[3]/div/div[1]/div[1]/form/div[2]/p/input')
searching.send_keys('大数据爬虫')
time.sleep(3)
searching.send_keys(Keys.ENTER)
time.sleep(10)
for i in range(3):titles = driver.find_elements(By.XPATH, '//span[@class="job-name"]')prices = driver.find_elements(By.XPATH, '//span[@class="salary"]')for title, price in zip(titles, prices):print(f'正在爬取第{i+1}页......')print(f'职位名称：{title.text}')print(f'薪资：{price.text}')search_next = driver.find_element(By.XPATH, '//i[@class="ui-icon-arrow-right"]').click()time.sleep(3)time.sleep(10)
driver.quit()
