l = [i.groupdict() for i in p.finditer(content)]
l=l[1:]
if len(l):return l
else:return None
def pageinfo(src):
'''
获取一个页面的详细信息
返回对于的字典列表
name:图片的名字
cutaddr：缩小的浏览图
picaddr：实际图片的地址
'''
d=os.path.split(src)[0]
try:
url = urllib2.urlopen(src)
content = url.read()#.decode('utf-8')
except:
print 'error'
return None
#find all the pictures info in a page
p = re.compile(r'<ul.*?>(.*?)</ul>',re.S)
r = p.findall(content)
if not r: return None
r = r[1]
p = re.compile(r'<li><a href="(?P<picaddr>.*?)".*?><img.*?alt="(?P<name>.*?)" *src="(?P<cutaddr>.*?)" */></a>.*?</li>')
l = [ i.groupdict() for i in p.finditer(r)]
for i in l:
i['picaddr']=d+'/'+i['picaddr']
if len(l): return l
else: return None
def nextpageaddr(src):
'''
从页面的html源码中获取下一个页面地址的名称，最后一页返回None
'''
content=readsrc(src)
p = re.compile(r'<a class="next" href="(.*?)">.*?</a>')
r = p.search(content)
if r:
return os.path.dirname(src)+"/"+r.group(1)
else:
return None
def picinfoaddr(src):
'''
参数相册图集的html代码
返回全部图片的相对地址
'''
content=readsrc(src)
p = re.compile(r'<div class="picinfo">.*?<a href="(?P<addr>.*?)".*?>.*?</div>',re.S)
r = p.search(content)
if r:
return os.path.dirname(src)+"/"+r.group(1)
else:
return None
def parseinfo(content):
'''
读取全部图片html代码，获得一个相册的详细信息
kw：关键字
title：标题
type：类型
pic：各个图片的地址列表，末尾加上_220x165，_medium,_small 可以得到不同大小的图片
'''
info={}
temp=str()
#title
temp=''
r=re.search('<h1>(.*?)</h1>',content)#get the pic title
if r:
temp = r.group(1)
info['title']=temp
#keyword
temp=''
r=re.search('<meta name="keywords" content="(.*?)" />',content)
if r:
temp = r.group(1)
info['kw']=temp
#type
r=re.findall('<i><a.*?>(.*?)</a></i>.*?>',content)
if r:
info['type']=':'.join(r)
else:
info['type']=''
r=re.search('<ul class=".*?">(.*?)</ul>',content,re.S)
if not r:return None
content=r.group(1)#filter content
# print content
r=re.findall('<a href=".*?<img.*?src="(.*?)".*?</a>',content)
for index,i in enumerate(r):
r[index]=i[0:i.rfind('_')]
# print r[index]
info['pic']=r
return info
import threading
class mthread(threading.Thread):
def __init__(self,tp,addr,lock):
threading.Thread.__init__(self)
# self.doc = minidom.Document()
self.doc=minidom.Document()
self.tp=tp
self.lock=lock
self.addr=addr
self.thread_stop=False
self.picdoc=None
def run(self):
self.picdoc = self.doc.createElement('urlclass')
# print self.tp
self.picdoc.setAttribute('type',self.tp)
# self.doc.appendChild(self.picdoc)
m=pageinfo(self.addr)
while self.addr:
for i in m:
# print i['picaddr']
picaddr=picinfoaddr(i['picaddr'])
# print picaddr
info=parseinfo(readsrc(picaddr))
name=info['title']
picture=doc.createElement('picture')
&nb