# coding=utf-8
'''created on 2023年8月16日
@author: lihhz
'''from spider.url_manager import urlmanager
from spider.html_********** import html**********
from spider.html_parser import htmlparser
import logging
import thread
logging.basicconfig(level=logging.debug,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %y %h:%m:%s',
filemode='w')
urlmanager = urlmanager();
html********** = html**********('d:/test1')
htmlparser = htmlparser()
url = '周杰倫&pn=%s&gsm=&ct=&ic=0&lm=-1&width=0&height=0'
class
main
(object):
def__init__
(self,rooturl):
self.rooturl = rooturl
defdownload
(self,u):
htmlcontent = html**********.downloadhtml(u)
imageurls = htmlparser.parsehtml('',htmlcontent)
html**********.downloadimage(imageurls)
defcraw
(self):
for i in range(0,100):#[::-1]:
u = url % (i*20)
logging.info('*****=%s' % (u))
thread.start_new_thread(self.download, (u,))
if __name__ == '__main__':
main = main('周杰倫&pn=0&gsm=50&ct=&ic=0&lm=-1&width=0&height=0')
main.craw()
pydev debugger: starting (pid: 10112)
traceback (most recent call last):
traceback (most recent call last):
traceback (most recent call last):
file "_pydevd_bundle\pydevd_cython_win32_27_32.pyx", line 918, in _pydevd_bundle.pydevd_cython_win32_27_32.threadtracer.__call__ (_pydevd_bundle/pydevd_cython_win32_27_32.c:15143)
traceback (most recent call last):
file "_pydevd_bundle\pydevd_cython_win32_27_32.pyx", line 918, in _pydevd_bundle.pydevd_cython_win32_27_32.threadtracer.__call__ (_pydevd_bundle/pydevd_cython_win32_27_32.c:15143)
file "_pydevd_bundle\pydevd_cython_win32_27_32.pyx", line 918, in _pydevd_bundle.pydevd_cython_win32_27_32.threadtracer.__call__ (_pydevd_bundle/pydevd_cython_win32_27_32.c:15143)
file "_pydevd_bundle\pydevd_cython_win32_27_32.pyx", line 918, in _pydevd_bundle.pydevd_cython_win32_27_32.threadtracer.__call__ (_pydevd_bundle/pydevd_cython_win32_27_32.c:15143)
unhandled exception in thread started by
sys.excepthook is missing
lost sys.stderr
unhandled exception in thread started by
sys.excepthook is missing
lost sys.stderr
unhandled exception in thread started by
sys.excepthook is missing
lost sys.stderr
# coding=utf-8
'''created on 2023年8月16日
@author: lihhz
'''from spider.url_manager import urlmanager
from spider.html_********** import html**********
from spider.html_parser import htmlparser
import logging
import thread
logging.basicconfig(level=logging.debug,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %y %h:%m:%s',
filemode='w')
urlmanager = urlmanager();
html********** = html**********('d:/test1')
htmlparser = htmlparser()
url = '周杰倫&pn=%s&gsm=&ct=&ic=0&lm=-1&width=0&height=0'
class
main
(object):
def__init__
(self,rooturl):
self.rooturl = rooturl
defdownload
(self,u,lock):
htmlcontent = html**********.downloadhtml(u)
imageurls = htmlparser.parsehtml('',htmlcontent)
# urlmanager.add_new_urls(notimageurls)
html**********.downloadimage(imageurls)
lock.release()#在子執行緒中釋放鎖
defcraw
(self):
locks = ;
for i in range(0,100):#[::-1]:
u = url % (i*20)
logging.info('*****=%s' % (u))
# 設計子執行緒的鎖
lock = thread.allocate_lock() # 分配鎖
lock.acquire()#獲取鎖
thread.start_new_thread(self.download, (u,lock,))
#在主線程中判斷是否所有的鎖都已釋放
for lock in locks:
while lock.locked():
pass
if __name__ == '__main__':
main = main('周杰倫&pn=0&gsm=50&ct=&ic=0&lm=-1&width=0&height=0')
main.craw()
python多執行緒 python多執行緒
通常來說，多程序適用於計算密集型任務，多執行緒適用於io密集型任務，如網路爬蟲。關於多執行緒和多程序的區別，請參考這個 下面將使用python標準庫的multiprocessing包來嘗試多執行緒的操作，在python中呼叫多執行緒要使用multiprocessing.dummy，如果是多程序則去掉...
python多執行緒詳解 Python多執行緒詳解
前言 由於最近的工作中一直需要用到python去處理資料，而在面對大量的資料時，python多執行緒的優勢就展現出來了。因而藉此機會，盡可能詳盡地來闡述python多執行緒。但對於其更底層的實現機制，在此不做深究，僅是對於之前的一知半解做個補充，也希望初學者能夠通過這篇文章，即便是照葫蘆畫瓢，也能夠...
python程式多執行緒 PYTHON多執行緒
在單執行緒的情況下，程式是逐條指令順序執行的。同一時間只做乙個任務，完成了乙個任務再進行下乙個任務。比如有5個人吃飯，單執行緒一次只允許乙個人吃，乙個人吃完了另乙個人才能接著吃，假如每個人吃飯都需要1分鐘，5個人就需要5分鐘。多執行緒的情況下，程式就會同時進行多個任務，雖然在同一時刻也只能執行某個任...