txt檔案匯入redis 單執行緒 多執行緒

一、txt檔案匯入redis-單執行緒

#!/bin/python

import redis
r=redis.redis("xx:xx:xx:xx")
file_object = open('x.txt')
try:
for line in file_object:
r.sadd('r_home_shallow_user',line.strip()) 
print line.strip()
finally:
file_object.close()
print "success!"

# -*-* encoding:utf-8 -*-

#!/bin/python
import redis
import sys
import threading
import time
# 判斷key是否存在、存在則刪除
def judge_key(redisip, rediskey):
r=redis.redis(redisip)
if r.exists(rediskey):
print "key exists, delete it!:", r.exists(rediskey)
try:
r.delete(rediskey)
time.sleep(30)
except exception as e:
print e
print "wait and try again!"
time.sleep(30)
judge_key(redisip, rediskey)
else:
time.sleep(10)
# 統計txt檔案的行數
def rowcount(filename):
count=0
thefile=open(filename)
while true:
buffer=thefile.read(1024*8192)
if not buffer:
break
count+=buffer.count('\n')
thefile.close()
print "row count:"
print(count)
return count
# 將txt檔案轉為list、並拆分成多個子list
def splitdata(count, splitrow, splitcnt, filename):
loaddataset = [line.strip() for line in open(filename).readlines()]
data = 
for i in range(splitcnt):
start = splitrow * i
end = splitrow * (i+1)
if end > count:
end = count
print len(loaddataset[start:end])
print loaddataset[start:end][0:3]
print "len(data):"
print len(data)
return data
# 多執行緒
class mythread(threading.thread):
def __init__(self,threadid,name,counter,data,redisip,rediskey):
threading.thread.__init__(self)
self.threadid = threadid
self.name = name
self.counter = counter
self.data = data
self.redisip = redisip
self.rediskey = rediskey
def run(self):
print "開始執行緒:",self.name
toredis(self.data, self.redisip, self.rediskey)
def __del__(self):
print self.name,"執行緒結束！"
# 匯入redis
def toredis(data, redisip, rediskey):
r=redis.redis(redisip)
try:
for line in data:
r.sadd(rediskey,line.strip())
finally:
print "success!"
# 多執行緒執行
def threads_run(splitcnt, data, redisip, rediskey):
threads = 
for tid in range(splitcnt):
t = mythread(tid, tid, 0, data[tid], redisip, rediskey)
t.start()
for i in range(splitcnt):
threads[i].join()
if __name__ == '__main__':
filename = sys.ar**[1]
splitrow = sys.ar**[2]
redisip = sys.ar**[3]
rediskey = sys.ar**[4]
# filename = 'x.txt'
# splitrow = 200000
# redisip = "xx.xx.xx.xx"
# rediskey = 'test:threading:list'
count = rowcount(filename)
splitcnt = count / splitrow + 1
data = splitdata(count, splitrow, splitcnt, filename)
# del key
judge_key(redisip, rediskey)
time.sleep(10)
judge_key(redisip, rediskey)
time.sleep(10)
threads_run(splitcnt, data, redisip, rediskey)

Redis單執行緒

redis 的單執行緒主要是指 redis 的網路 io 和鍵值對讀寫是由乙個執行緒來完成的，這也是 redis 對外提供鍵值儲存服務的主要流程。當多個客戶端發起命令，這些命令併發執行時，在redis內部，會排隊逐個執行，也就是執行命令的那個操作是由乙個執行緒執行的。但 redis 的其他功能，比如...

Redis單執行緒理解

簡介 從接觸redis到現在，一直被它的單執行緒問題困擾，這對於乙個苛求原理的我來說是種折磨，今天吃飯途中看了幾篇部落格，茅塞頓開。個人理解 redis分客戶端和服務端，一次完整的redis請求事件有多個階段 客戶端到伺服器的網路連線 redis讀寫事件發生 redis服務端的資料處理 單執行緒 資...

redis單執行緒模型

redis基於reactor模式開發了自己的網路事件處理器，稱之為檔案事件處理器 file event hanlder 檔案事件處理器由socket io多路復用程式 檔案事件分派器 dispather 事件處理器 handler 四部分組成。io多路復用程式會同時監聽多個socket，當被監聽的s...