Python爬虫:下载人生格言

爬取网页

将这些格言下载存储到本地

代码:

<code>import requests #导入requests库，用于提取网页

from lxml import etree#导入lxml库，用于Xpath数据解析

#请求头

header={

'user-agent':

'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0'

}#每个浏览器的user-agent不一样，在浏览器中查找

url="http://m.3chongmen.com/renshenggeyan/162.html"#请求网址code>

res1=requests.get(url=url,headers=header).text

html=etree.HTML(res1)

title=html.xpath('//div[@class="title"]/h1/text()')[0]#数据解析，提取标题code>

content=html.xpath('//div[@class="content"]/text()')#数据解析，提取内容code>

content="".join(content)code>

print(title)

print(content)

运行结果:

<code>import requests

from lxml import etree

header={

'user-agent':

'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0'

}

def spider(url):

res1=requests.get(url=url,headers=header).text

html=etree.HTML(res1)

content=html.xpath('//div[@class="content"]/text()')code>

content="".join(content)code>

title=html.xpath('//div[@class="title"]/h1/text()')[0]code>

return title,content

url1="http://m.3chongmen.com/renshenggeyan"code>

res=requests.get(url=url1,headers=header).text

html=etree.HTML(res)

links=html.xpath('//ul[@class="list_cnt"]//a[@target="_blank"]/@href')code>

for link in links:

title,content=spider(link)

with open(f'格言/{ title}.txt','w',encoding='utf-8') as f:code>

f.write(title+'\n\n')

f.write(content)

运行结果: