某直聘python岗位

 1 from selenium import webdriver
 2 from selenium.webdriver.common.by import By
 3 from selenium.webdriver.support import expected_conditions as EC
 4 from selenium.webdriver.support.ui import WebDriverWait
 5 import csv
 6 import time
 7 import random
 8 import requests
 9 import parsel
10 
11 f = open('boss.csv', mode='a', encoding='utf-8-sig', newline='')
12 csvWriter = csv.DictWriter(f, fieldnames=[
13     '标题',
14     '地区',
15     '薪资',
16     '经验',
17     '公司名',
18     '公司领域',
19     '福利',
20     '公司规模',
21     '详情页',
22     # '所需技能',
23 ])
24 csvWriter.writeheader() #  写入头
25 
26 url = 'https://www.zhipin.com/c100010000/?query=python&ka=sel-city-100010000'
27 driver = webdriver.Chrome()
28 driver.get(url=url)
29 driver.implicitly_wait(10)
30 driver.maximize_window() # 最大化窗口
31 # lis = driver.find_elements_by_css_selector('.job-list ul li')
32 
33 def get_job_details():
34     lis = driver.find_elements(By.CSS_SELECTOR, '.job-list ul li')
35     for li in lis:
36         title = li.find_element(By.CSS_SELECTOR, '.job-name a').text # 标题
37         area = li.find_element(By.CSS_SELECTOR, '.job-area').text # 地区
38         salary = li.find_element(By.CSS_SELECTOR, '.job-limit .red').text # 薪资
39         experience = li.find_element(By.CSS_SELECTOR, '.job-limit p').text # 工作经验
40         companyName = li.find_element(By.CSS_SELECTOR, '.company-text h3 a').text # 公司名
41         companyStyle = li.find_element(By.CSS_SELECTOR, '.company-text p a').text # 公司领域
42         welfare = li.find_element(By.CSS_SELECTOR, '.info-desc').text # 公司福利
43         peopleInclude = li.find_element(By.CSS_SELECTOR, '.company-text p em').text # 公司规模
44         detailPage = li.find_element(By.CSS_SELECTOR, '.job-name a').get_attribute('href') # 详情页
45         # skillsNeed = get_skills_desc(desc_url=detailPage) # 加载窗口
46         print(title, area, salary, experience, companyName, companyStyle, peopleInclude, detailPage, skillsNeed, sep=' | ')
47         dit = {
48             '标题': title,
49             '地区': area,
50             '薪资': salary,
51             '经验': experience,
52             '公司名': companyName,
53             '公司领域': companyStyle,
54             '福利': welfare,
55             '公司规模': peopleInclude,
56             '详情页': detailPage,
57             # '所需技能': skillsNeed,
58         }
59         csvWriter.writerow(dit) # 逐行写入
60 
61 # def get_skills_desc(desc_url):
62 #     # 获取详情页所需技能的函数
63 #     # 加载另一个窗口
64 #     js = 'window.open()'
65 #     driver.execute_script(js) # 打开新窗口
66 #     driver.switch_to.window(window_name=driver.window_handles[-1]) # 切换到刚打开的窗口
67 #     # 开始加载url
68 #     driver.get(url=desc_url) # 加载详情页url
69 #     driver.implicitly_wait(10) # 等待网页加载完成
70 #     skills_need = driver.find_element(By.CSS_SELECTOR, '.text') # 直接查找描述字段
71 #     return skills_need
72 
73 for page in range(1, 100 + 1):
74     print(f'------------------------正在爬取第{page}页内容----------------------------')
75     time.sleep(random.uniform(2,5))
76     get_job_details() # 获取信息
77     next_button = driver.find_element(By.CSS_SELECTOR, '.page .next')
78     if next_button: # 如果找到，就表示有下一页
79         next_button.click() # 点击下一页
80     else:
81         print(f'已经没有了！ 第{page}已经是最后一页！')
82 
83 driver.quit() # 退出浏览器