一.開發環境
本文執行環境為windows10+python3.7
使用的第三方庫有selenium（操作瀏覽器）+pymysql（資料庫）+bs4（解析）+chrome+chromedriver
二.先決條件
利用京東的搜尋結果，然後把結果儲存起來
那麼接下來就是找到京東搜尋的相關url。因為可見即可爬。
base_url = ""
params =
browser.get(base_url + urlencode(params))
三.完整**
所以到這裡，所有的思路就是：
1）先找到要爬取的介面
2）分析介面dom，找到需要的資訊
3）解析
4）儲存到資料庫
第一步，先建立資料庫
def init_database():
create_table_sql = "create table if not exists jd_android (id int not null, name varchar(255) not null)"
try:
db = pymysql.connect(host="localhost", user="root", password="root", port=3306, db="spiders")
cursor = db.cursor()
cursor.execute(create_table_sql)
finally:
if db:
db.close()
def spider_by_chrome():
browser = webdriver.chrome()
db = pymysql.connect(db="spiders", user="root", password="root", port=3306)
for index in range(10):
get_source_page(browser, index, db)
browser.close()
第三步，解析並儲存到資料庫
def get_source_page(browser, offset, db):
''':param browser: the browser we use to spider
:param offset: the page to search
:param db: database to store data
:return: the final html page
'''params =
browser.get(base_url + urlencode(params))
# input = browser.find_element_by_id("key")
# input.send_keys("android")
# input.send_keys(keys.enter)
wait = webdriverwait(browser, 10)
wait.until(ec.presence_of_element_located((by.id, "j_searchwrap")))
html = beautifulsoup(browser.page_source, "lxml")
for item in html.find_all("li", class_="gl-item"):
book_name = item.select("em")[1]
print(book_name.text)
global book_id
cursor = db.cursor()
cursor.execute("insert into jd_android (id, name) values (%s, %s)", (book_id, book_name.text))
book_id = book_id + 1
# item_xml = beautifulsoup(item, "lxml")
# print(item_xml.find_all("em"))
db.commit()#儲存
綜上，是基本的爬蟲操作。
四.完整**
from selenium import webdriver
from selenium.webdriver.common.by import by
from selenium.webdriver.common.keys import keys
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.wait import webdriverwait
from urllib.parse import urlencode
import pymysql
from bs4 import beautifulsoup
base_url = ""
book_id = 1
def get_source_page(browser, offset, db):
''':param browser: the browser we use to spider
:param offset: the page to search
:param db: database to store data
:return: the final html page
'''params =
browser.get(base_url + urlencode(params))
# input = browser.find_element_by_id("key")
# input.send_keys("android")
# input.send_keys(keys.enter)
wait = webdriverwait(browser, 10)
wait.until(ec.presence_of_element_located((by.id, "j_searchwrap")))
html = beautifulsoup(browser.page_source, "lxml")
for item in html.find_all("li", class_="gl-item"):
book_name = item.select("em")[1]
print(book_name.text)
global book_id
cursor = db.cursor()
cursor.execute("insert into jd_android (id, name) values (%s, %s)", (book_id, book_name.text))
book_id = book_id + 1
# item_xml = beautifulsoup(item, "lxml")
# print(item_xml.find_all("em"))
db.commit()
def spider_by_chrome():
browser = webdriver.chrome()
db = pymysql.connect(db="spiders", user="root", password="root", port=3306)
for index in range(10):
get_source_page(browser, index, db)
browser.close()
def init_database():
create_table_sql = "create table if not exists jd_android (id int not null, name varchar(255) not null)"
try:
db = pymysql.connect(user="root", password="root", port=3306, db="spiders")
cursor = db.cursor()
cursor.execute(create_table_sql)
finally:
if db:
db.close()
init_database()
spider_by_chrome()
python京東商品 Python爬取京東商品資料
對京東某一商品資訊頁面的html 進行分析。能夠發現它的圖書產品資訊頁面都含有這樣一段 不同類的商品頁面有些不同 window.pageconfig 顯然這就是我們須要的商品資訊，python 例如以下 import json import re import urllib for i in ran...
python爬蟲小練習 小說爬取
import requests from bs4 import beautifulsoup if name main headers 對首頁的頁面資料進行爬取 url page text requests.get url url,headers headers text 在首頁中解析出章節的標題和詳...
Python爬取京東商品
自動開啟京東首頁，並輸入你要搜尋的東西，進入介面進行爬取資訊 from selenium import webdriver import time class jdspider object def init self self.browser webdriver.chrome self.url s...