使用知識
使用說明
源**
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @time : 2020/1/16 15:34
# @author : wsx
# @site :
# @file : cars.py
# @software: pycharm
import json
from multiprocessing import pool
import requests
from requests.exceptions import requestexception
import re
from bs4 import beautifulsoup
defget_one_page
(url)
:"""
請求網頁函式.
:param url:
:return:
"""headers =
try:
response = requests.get(url, headers=headers)
print
(response.status_code)
if response.status_code ==
200:
return response.text
return
none
except requestexception:
return
none
defparse_one_page
(html, first_letter)
:"""
網頁處理函式, 生成器
:param html:
:param first_letter:
:return:iterable
"""# 載入網頁
soup = beautifulsoup(html,
'lxml'
)# 建立字典,儲存資料
info =
# 找出所需資訊在的標籤
branches = soup.find_all(
'dl'
)# 先獲取品牌
for branch in branches:
info[
'branch_name'
]= branch.dt.div.a.string.strip(
) info[
'branch_id'
]= branch[
'id'
] info[
'branch_first_letter'
]= first_letter
print
('正在抓取...品牌:'
, info[
'branch_name'])
# 生成新的處理塊
block = branch.find_all(
'dd'
) soup = beautifulsoup(
str(block)
,'lxml'
)# 獲取某一品牌下的所有製造商
producers = soup.find_all(
'div'
, attrs=
)for producer in producers:
info[
'producer'
]= producer.a.get_text(
).strip(
)# 找不到這個引數呀.
info[
'producer_id']=
''print
('正在抓取...生產商:'
, info[
'producer'])
cars = producer.find_next(
'ul'
)for car in cars.find_all(
'li'
, attrs=):
info[
'car_series_id'
]= car[
'id'
] info[
'car_series'
]= car.h4.a.get_text(
).strip(
)# **這個引數難提取, 初步過濾一下
price = car.find_all(
'a', attrs=
)# 判斷一下抓取的是不是**, 用正規表示式再過濾一下
if price:
print
(price[0]
.get_text())
if re.match(
'.*?萬.*?'
, price[0]
.get_text(
), re.s)
: info[
'car_price'
]= price[0]
.get_text(
).strip(
)else
: info[
'car_price']=
'暫無**'
# 做成迭代器
yield info
defwrite_file
(content)
:"""
將抓取資料儲存成json檔案
:param content:
:return: none
"""with
open
('cars.txt'
,'a'
, encoding=
'utf-8'
)as f:
f.write(json.dumps(content, ensure_ascii=
false)+
'\n'
) f.close(
)def
main
(first_letter)
:"""
主函式:param first_letter:
:return: none
"""html = get_one_page(
''+ first_letter +
'.html'
) soup = beautifulsoup(html,
'lxml'
) html = soup.prettify(
)# 測試時先存在本地以免頻繁訪問站點
# with open('car_home.html', 'w', encoding='utf-8') as f:
# f.write(html)
# f.close()
# with open('car_home.html', 'r', encoding='utf-8') as f:
# html = f.read()
# f.close()
for item in parse_one_page(html, first_letter)
: write_file(item)
if __name__ ==
'__main__'
:# 如不需要按照字母順序, 則uncomment
# pool = pool()
# pool.map(main, [chr(i + ord('a')) for i in range(26)])
# 如需要多執行緒, 則comment
for letter in
[chr
(i +
ord(
'a')
)for i in
range(26
)]: main(letter)
編寫**過程中遇到beautifulsoup中,find_all()方法如果只需要確定是否存在某個屬性,而不指定具體屬性值,可以寫成下面這樣:
car.find_all(
'a', attrs=
)
爬取汽車之家
爬汽車之家新聞 爬取汽車之家新聞 import requests 向汽車之家傳送get請求，獲取到頁面 ret requests.get print ret.text 用bs4解析 from bs4 import beautifulsoup 例項化得到物件，傳入要解析的文字，解析器 html.par...
Python練習 scrapy 爬取汽車之家文章
autohome.py spider檔案 coding utf 8 import scrapy from autohome.items import autohomeitem class autohomespider scrapy.spider name autohome allowed domai...
手寫爬取靜態頁面汽車之家
scrapy寫多了,手寫爬蟲有點生疏,今天來回顧手寫爬取靜態頁面,以便日後做筆記用,我今天爬取的是汽車之家網頁,第一步 匯入requests和bs4 import requests from bs4 import beautifulsoup 第三步 解析頁面,在這裡我們用的beautifulsoup...