Python实现多线程抓取妹子图


Posted in Python onAugust 08, 2015

心血来潮写了个多线程抓妹子图,虽然代码还是有一些瑕疵,但是还是记录下来,分享给大家。

Pic_downloader.py

# -*- coding: utf-8 -*-
"""
Created on Fri Aug 07 17:30:58 2015
 
@author: Dreace
"""
import urllib2
import sys
import time
import os
import random
from multiprocessing.dummy import Pool as ThreadPool 
type_ = sys.getfilesystemencoding()
def rename():
  return time.strftime("%Y%m%d%H%M%S")
def rename_2(name): 
  if len(name) == 2: 
    name = '0' + name + '.jpg' 
  elif len(name) == 1: 
    name = '00' + name + '.jpg' 
  else: 
    name = name + '.jpg' 
  return name
def download_pic(i):
  global count
  global time_out
  if Filter(i):
    try: 
      content = urllib2.urlopen(i,timeout = time_out)
      url_content = content.read()
      f = open(repr(random.randint(10000,999999999)) + "_" + rename_2(repr(count)),"wb")
      f.write(url_content)
      f.close()
      count += 1
    except Exception, e:
      print i + "下载超时,跳过!".decode("utf-8").encode(type_)
def Filter(content):
  for line in Filter_list:
    line=line.strip('\n')
    if content.find(line) == -1:
      return True
def get_pic(url_address):
  global pic_list
  try:
    str_ = urllib2.urlopen(url_address, timeout = time_out).read()
    url_content = str_.split("\"")
    for i in url_content:
      if i.find(".jpg") != -1:
        pic_list.append(i)  
  except Exception, e:
    print "获取图片超时,跳过!".decode("utf-8").encode(type_)
MAX = 2
count = 0
time_out = 60
thread_num = 30
pic_list = []
page_list = []
Filter_list = ["imgsize.ph.126.net","img.ph.126.net","img2.ph.126.net"]
dir_name = "C:\Photos\\"+rename()
os.makedirs(dir_name)
os.chdir(dir_name)
start_time = time.time()
url_address = "http://sexy.faceks.com/?page="
for i in range(1,MAX + 1): 
  page_list.append(url_address + repr(i))
page_pool = ThreadPool(thread_num)
page_pool.map(get_pic,page_list)
print "获取到".decode("utf-8").encode(type_),len(pic_list),"张图片,开始下载!".decode("utf-8").encode(type_)
pool = ThreadPool(thread_num) 
pool.map(download_pic,pic_list)
pool.close() 
pool.join()
print count,"张图片保存在".decode("utf-8").encode(type_) + dir_name
print "共耗时".decode("utf-8").encode(type_),time.time() - start_time,"s"

我们来看下一个网友的作品

#coding: utf-8 #############################################################
# File Name: main.py
# Author: mylonly
# mail: mylonly@gmail.com
# Created Time: Wed 11 Jun 2014 08:22:12 PM CST
#########################################################################
#!/usr/bin/python

import re,urllib2,HTMLParser,threading,Queue,time

#各图集入口链接
htmlDoorList = []
#包含图片的Hmtl链接
htmlUrlList = []
#图片Url链接Queue
imageUrlList = Queue.Queue(0)
#捕获图片数量
imageGetCount = 0
#已下载图片数量
imageDownloadCount = 0
#每个图集的起始地址,用于判断终止
nextHtmlUrl = ''
#本地保存路径
localSavePath = '/data/1920x1080/'

#如果你想下你需要的分辨率的,请修改replace_str,有如下分辨率可供选择1920x1200,1980x1920,1680x1050,1600x900,1440x900,1366x768,1280x1024,1024x768,1280x800
replace_str = '1920x1080'

replaced_str = '960x600'

#内页分析处理类
class ImageHtmlParser(HTMLParser.HTMLParser):
def __init__(self):
self.nextUrl = ''
HTMLParser.HTMLParser.__init__(self)
def handle_starttag(self,tag,attrs):
global imageUrlList
if(tag == 'img' and len(attrs) > 2 ):
if(attrs[0] == ('id','bigImg')):
url = attrs[1][1]
url = url.replace(replaced_str,replace_str)
imageUrlList.put(url)
global imageGetCount
imageGetCount = imageGetCount + 1
print url
elif(tag == 'a' and len(attrs) == 4):
if(attrs[0] == ('id','pageNext') and attrs[1] == ('class','next')):
global nextHtmlUrl
nextHtmlUrl = attrs[2][1];

#首页分析类
class IndexHtmlParser(HTMLParser.HTMLParser):
def __init__(self):
self.urlList = []
self.index = 0
self.nextUrl = ''
self.tagList = ['li','a']
self.classList = ['photo-list-padding','pic']
HTMLParser.HTMLParser.__init__(self)
def handle_starttag(self,tag,attrs):
if(tag == self.tagList[self.index]):
for attr in attrs:
if (attr[1] == self.classList[self.index]):
if(self.index == 0):
#第一层找到了
self.index = 1
else:
#第二层找到了
self.index = 0
print attrs[1][1]
self.urlList.append(attrs[1][1])
break
elif(tag == 'a'):
for attr in attrs:
if (attr[0] == 'id' and attr[1] == 'pageNext'):
self.nextUrl = attrs[1][1]
print 'nextUrl:',self.nextUrl
break

#首页Hmtl解析器
indexParser = IndexHtmlParser()
#内页Html解析器
imageParser = ImageHtmlParser()

#根据首页得到所有入口链接
print '开始扫描首页...'
host = 'http://desk.zol.com.cn'
indexUrl = '/meinv/'
while (indexUrl != ''):
print '正在抓取网页:',host+indexUrl
request = urllib2.Request(host+indexUrl)
try:
m = urllib2.urlopen(request)
con = m.read()
indexParser.feed(con)
if (indexUrl == indexParser.nextUrl):
break
else:
indexUrl = indexParser.nextUrl
except urllib2.URLError,e:
print e.reason

print '首页扫描完成,所有图集链接已获得:'
htmlDoorList = indexParser.urlList

#根据入口链接得到所有图片的url
class getImageUrl(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
for door in htmlDoorList:
print '开始获取图片地址,入口地址为:',door
global nextHtmlUrl
nextHtmlUrl = ''
while(door != ''):
print '开始从网页%s获取图片...'% (host+door)
if(nextHtmlUrl != ''):
request = urllib2.Request(host+nextHtmlUrl)
else:
request = urllib2.Request(host+door)
try:
m = urllib2.urlopen(request)
con = m.read()
imageParser.feed(con)
print '下一个页面地址为:',nextHtmlUrl
if(door == nextHtmlUrl):
break
except urllib2.URLError,e:
print e.reason
print '所有图片地址均已获得:',imageUrlList

class getImage(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
global imageUrlList
print '开始下载图片...'
while(True):
print '目前捕获图片数量:',imageGetCount
print '已下载图片数量:',imageDownloadCount
image = imageUrlList.get()
print '下载文件路径:',image
try:
cont = urllib2.urlopen(image).read()
patter = '[0-9]*\.jpg';
match = re.search(patter,image);
if match:
print '正在下载文件:',match.group()
filename = localSavePath+match.group()
f = open(filename,'wb')
f.write(cont)
f.close()
global imageDownloadCount
imageDownloadCount = imageDownloadCount + 1
else:
print 'no match'
if(imageUrlList.empty()):
break
except urllib2.URLError,e:
print e.reason
print '文件全部下载完成...'

get = getImageUrl()
get.start()
print '获取图片链接线程启动:'

time.sleep(2)

download = getImage()
download.start()
print '下载图片链接线程启动:'

批量抓取指定网页上的所有图片

# -*- coding:utf-8 -*-
# coding=UTF-8
 
import os,urllib,urllib2,re
 
url = u"http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=index&fr=&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=python&oq=python&rsp=-1"
outpath = "t:\\"
 
def getHtml(url):
  webfile = urllib.urlopen(url)
  outhtml = webfile.read()
  print outhtml
  return outhtml
 
def getImageList(html):
  restr=ur'('
  restr+=ur'http:\/\/[^\s,"]*\.jpg'
  restr+=ur'|http:\/\/[^\s,"]*\.jpeg'
  restr+=ur'|http:\/\/[^\s,"]*\.png'
  restr+=ur'|http:\/\/[^\s,"]*\.gif'
  restr+=ur'|http:\/\/[^\s,"]*\.bmp'
  restr+=ur'|https:\/\/[^\s,"]*\.jpeg'  
  restr+=ur'|https:\/\/[^\s,"]*\.jpeg'
  restr+=ur'|https:\/\/[^\s,"]*\.png'
  restr+=ur'|https:\/\/[^\s,"]*\.gif'
  restr+=ur'|https:\/\/[^\s,"]*\.bmp'
  restr+=ur')'
  htmlurl = re.compile(restr)
  imgList = re.findall(htmlurl,html)
  print imgList
  return imgList
 
def download(imgList, page):
  x = 1
  for imgurl in imgList:
    filepathname=str(outpath+'pic_%09d_%010d'%(page,x)+str(os.path.splitext(urllib2.unquote(imgurl).decode('utf8').split('/')[-1])[1])).lower()
    print '[Debug] Download file :'+ imgurl+' >> '+filepathname
    urllib.urlretrieve(imgurl,filepathname)
    x+=1
 
def downImageNum(pagenum):
  page = 1
  pageNumber = pagenum
  while(page <= pageNumber):
    html = getHtml(url)#获得url指向的html内容
    imageList = getImageList(html)#获得所有图片的地址,返回列表
    download(imageList,page)#下载所有的图片
    page = page+1
 
if __name__ == '__main__':
  downImageNum(1)

以上就是给大家汇总的3款Python实现的批量抓取妹纸图片的代码了,希望对大家学习Python爬虫能够有所帮助。

Python 相关文章推荐
Python警察与小偷的实现之一客户端与服务端通信实例
Oct 09 Python
Python中type的构造函数参数含义说明
Jun 21 Python
Python的Scrapy爬虫框架简单学习笔记
Jan 20 Python
详解Python编程中对Monkey Patch猴子补丁开发方式的运用
May 27 Python
Python机器学习之SVM支持向量机
Dec 27 Python
Python实现求解括号匹配问题的方法
Apr 17 Python
python学生管理系统
Jan 30 Python
Python实现插入排序和选择排序的方法
May 12 Python
Python简单处理坐标排序问题示例
Jul 11 Python
Python检查 云备份进程是否正常运行代码实例
Aug 22 Python
基于Django实现日志记录报错信息
Dec 17 Python
Python requests库参数提交的注意事项总结
Mar 29 Python
通过Python来使用七牛云存储的方法详解
Aug 07 #Python
Python爬虫框架Scrapy实战之批量抓取招聘信息
Aug 07 #Python
深入理解Python中命名空间的查找规则LEGB
Aug 06 #Python
举例详解Python中yield生成器的用法
Aug 05 #Python
Python中return语句用法实例分析
Aug 04 #Python
python函数形参用法实例分析
Aug 04 #Python
Python简明入门教程
Aug 04 #Python
You might like
phpmail类发送邮件函数代码
2012/02/20 PHP
PHP Parse Error: syntax error, unexpected $end 错误的解决办法
2012/06/05 PHP
php上传apk后自动提取apk包信息的使用(示例下载)
2013/04/26 PHP
php函数指定默认值方法的小例子
2013/12/04 PHP
PHP lcfirst()函数定义与用法
2019/03/08 PHP
Swoole源码中如何查询Websocket的连接问题详解
2020/08/30 PHP
[原创]提供复制本站内容时出现,该文章转自脚本之家等字样的js代码
2007/03/27 Javascript
Javascript 继承机制实例
2009/08/12 Javascript
javascript中的return和闭包函数浅析
2014/06/06 Javascript
基于jQuery实现的无刷新表格分页实例
2016/02/17 Javascript
Javascript自执行匿名函数(function() { })()的原理浅析
2016/05/15 Javascript
详谈js使用in和hasOwnProperty获取对象属性的区别
2017/04/25 Javascript
es6+angular1.X+webpack 实现按路由功能打包项目的示例
2017/08/16 Javascript
vue router使用query和params传参的使用和区别
2017/11/13 Javascript
150行Node.js实现的dns代理工具
2019/08/02 Javascript
vue项目中锚点定位替代方式
2019/11/13 Javascript
element-ui table行点击获取行索引(index)并利用索引更换行顺序
2020/02/27 Javascript
python2.7读取文件夹下所有文件名称及内容的方法
2018/02/24 Python
Flask框架实现给视图函数增加装饰器操作示例
2018/07/16 Python
python实现键盘控制鼠标移动
2020/11/27 Python
python实现连续图文识别
2018/12/18 Python
windows下搭建python scrapy爬虫框架步骤
2018/12/23 Python
Python Pandas 如何shuffle(打乱)数据
2019/07/30 Python
PyTorch中topk函数的用法详解
2020/01/02 Python
浅谈python输出列表元素的所有排列形式
2020/02/26 Python
纯CSS打造(无图像无js)的非常流行的讲话(语音)气泡效果
2012/12/28 HTML / CSS
CSS3制作炫酷带方向感应的鼠标滑过图片3D动画
2016/03/16 HTML / CSS
售后服务经理岗位职责范本
2014/02/22 职场文书
党的群众路线教育实践活动方案
2014/10/31 职场文书
2014年党委工作总结
2014/11/22 职场文书
2015年毕业实习工作总结
2015/05/29 职场文书
2015大学迎新晚会主持词
2015/07/16 职场文书
2016秋季运动会前导词
2015/11/25 职场文书
高三物理教学反思
2016/02/20 职场文书
新学期小学班主任工作计划
2019/06/21 职场文书
使用Docker容器部署rocketmq单机的全过程
2022/04/03 Servers