1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192 |
- # coding=utf-8
- '''
- Created on 2017年7月1日
- @vsersion:python3.6
- @author: liuyuqi
- '''
- import random
- import time
- # 导入开发模块
- import requests
- from bs4 import BeautifulSoup
- from nt import chdir
- class Lianjia(object):
- def __init__(self):
- super(Lianjia, self).__init__()
- self.session = requests.Session()
- headers = {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
- 'Referer': 'http://sh.lianjia.com/ershoufang/',
- }
- self.urls=[]
- self.session.headers.update(headers)
- def getUrls(self):
- # 基于for循环,构造完整的爬虫链接
- for city in citys:
- url = 'http://sh.lianjia.com/ershoufang/%s/' % city
- res = self.session.get(url) # 发送get请求
- res = res.text.encode(res.encoding).decode('utf-8') # 需要转码,否则会有问题
- soup = BeautifulSoup(res, 'html.parser') # 使用bs4模块,对响应的链接源代码进行html解析
- page = soup.findAll('div', {'class': 'c-pagination'}) # 使用finalAll方法,获取指定标签和属性下的内容
- pages = [i.strip() for i in page[0].text.split('\n')] # 抓取出每个区域的二手房链接中所有的页数
- if len(pages) > 3:
- total_pages = int(pages[-3])
- else:
- total_pages = int(pages[-2])
- for j in list(range(1, total_pages + 1)): # 拼接所有需要爬虫的链接
- self.urls.append('http://sh.lianjia.com/ershoufang/%s/d%s' % (city, j))
- # 随机睡眠2-10s
- time.sleep(random.randint(2, 10))
- def mSpider(self):
- # 获取所有url
- self.getUrls()
- # 创建csv文件,用于后面的保存数据
- file = open('链家二手房.csv', 'w', encoding='utf-8')
- for url in self.urls: # 基于for循环,抓取出所有满足条件的标签和属性列表,存放在find_all中
- res = requests.get(url)
- res = res.text.encode(res.encoding).decode('utf-8')
- soup = BeautifulSoup(res, 'html.parser')
- find_all = soup.find_all(name='div', attrs={'class': 'info'})
- for i in list(range(len(find_all))): # 基于for循环,抓取出所需的各个字段信息
- title = find_all[i].find('a')['title'] # 每套二手房的标语
- res2 = find_all[i]
- name = res2.find_all('div', {'class': 'where'})[0].find_all('span')[0].text # 每套二手房的小区名称
- room_type = res2.find_all('div', {'class': 'where'})[0].find_all('span')[1].text # 每套二手房的户型
- size = res2.find_all('div', {'class': 'where'})[0].find_all('span')[2].text[:-3] # 每套二手房的面积
- # 采用列表解析式,删除字符串的首位空格
- info = [i.strip() for i in res2.find_all('div', {'class': 'con'})[0].text.split('\n')]
- region = info[1] # 每套二手房所属的区域
- loucheng = info[2][2:] # 每套二手房所在的楼层
- chaoxiang = info[5][2:] # 每套二手房的朝向
- builtdate = info[-3][2:] # 每套二手房的建筑时间
- # 每套二手房的总价
- price = find_all[i].find('div', {'class': 'price'}).text.strip()[:-1]
- # 每套二手房的平方米售价
- price_union = find_all[i].find('div', {'class': 'price-pre'}).text.strip()[:-3]
- # print(name,room_type,size,region,loucheng,chaoxiang,price,price_union,builtdate)
- # 将上面的各字段信息值写入并保存到csv文件中
- file.write(','.join(
- (name, room_type, size, region, loucheng, chaoxiang, price, price_union, builtdate)) + '\n')
- # 关闭文件(否则数据不会写入到csv文件中)
- file.close()
- if __name__ == "__main__":
- workSpace = "E:\\data\\workspace\\Python_Tools\\lianjianalysis"
- chdir(workSpace)
- # 指定爬虫所需的上海各个区域名称
- citys = ['pudongxinqu','minhang','baoshan','xuhui','putuo','yangpu','changning','songjiang',
- 'jiading','huangpu','jinan','zhabei','hongkou','qingpu','fengxian','jinshan','chongming']
- lianjia=Lianjia()
- lianjia.mSpider(citys)
|