利用xpath和正则对链家网站北京市东城区、海淀区、丰台区的租房房源前50页数据进行爬取。数据包括链家网站链家网站北京市东城区、海淀区、丰台区的租房房源数据共4500条，包括房源信息标题、区域、地点、房屋名称、建筑面积、朝向、户型、价格、详情页链接等信息。
import requestsimport jsonfrom lxml import etreeimport redef get_info(zone):for i in range(1, 51):headers = {"user-agent": "# 自己的user-agent"}url = '/zufang/'+zone+'/pg'+str(i)+'/#contentList'url_total = '/zufang/'+zone# get or post 看请求Headershtml = requests.get(url, headers=headers)html = html.textsoup = etree.HTML(html)# 标题title = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[1]/a//text()')title_new = []for i in title:i = i.strip('\n') # 删除换行符i = i.strip(' ') #删除空格title_new.append(i)# print(title_new)# 所在区域（海淀、丰台、东城）zones = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/a[1]//text()')# 地点place = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/a[2]//text()')# print(place)# 具体名称name = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/a[3]//text()')# print(name)# 面积size = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]//text()[5]')size_new = []for i in size:i = i.strip('\n') # 删除换行符，但是只能删除第一个换行符i = i.replace('\n', '') # 利用replace将换行符替换i = i.strip(' ')size_new.append(i)# print(size_new)# 朝向orientation = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/text()[6]')orientation_new = []for i in orientation:i = i.strip(' ')orientation_new.append(i)# print(orientation_new)# 户型type = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/text()[7]')type_new = []for i in type:i = i.strip('\n')i = i.strip(' ')type_new.append(i)# print(type_new)# 价格price = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/span/em//text()')# print(price)详情页链接house_url = re.findall('<a class="twoline" target="_blank" href="(/zufang/.*?.html)">', html)# print(house_url)# 将爬取信息存入excelfor i in range(len(title)):f = open('zufang.csv', 'a', encoding='utf8')f.write('{},{},{},{},{},{},{},{},{}\n'.format(title_new[i], zones[i], place[i], name[i], size_new[i], orientation_new[i], type_new[i], price[i], url_total+house_url[i]))f.close()if __name__ == '__main__':f = open('zufang.csv', 'a', encoding='utf8')f.write('title,zone,place,name,size,orientation,type,price,link\n') # 写入表头f.close()for zone in ('dongcheng', 'haidian', 'fengtai'):get_info(zone)
