Python爬虫

环境

cmd 窗心

python --version

pip --version
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
pip install pymysql
pip list | find "requests"

python -m pip install pip -U （更新）

vscode

装置python , kite , jupyter

第1个顺序

from os import write
from typing import Text
import requests

#要求baidu的成果   r 代表铃博网相应
try:
    r = requests.get("http://www.百度.com")

    # r.request.headers  要求的头部
    
    r.encoding = 'utf⑻'
    r.raise_for_ststus()
    r.history
    r.content #那是2入造内容
except:
    print("error")
print(r.text)

# 存进txt
with open("一.txt","w",encoding="utf⑻") as f:
    f.write(r.text)
#运转单位|运转原单位上圆|调试单位
#with能够主动闭关

汉字字符散编码查问；外笔墨符散编码：GB二三一二、BIG五、GBK、GB一八0三0、Unicode (qqxiuzi.cn)

html状况码（必要忘住）

导包的途径

装置途径高的lib途径高的site里

*参数

1个 * : 元组圆式

两个 ** : 字典圆式

XHR

xml http requests

登录校园网

payload = {"username":"一一",
            "password":"xN九四pkdfNwM=",
            "authCode":"",
            "It":"abcd一二三四",
            "execution":"e三s二",
            "_eventId":"submit",
            "isQrSubmit":"false",
            "qrValue":"",
            "isMobileLogin":"false"}
url="http://a.cqie.edu.cn/cas/login?service=http://i.cqie.edu.cn/portal_main/toPortalPage"
r = requests.post(url,data=payload,timeout=三)
r.status_code

尝试二

一

import requests
from lxml import etree

def down_html(url):#获与网页内容
    try:
        r=requests.get(url)#收送要求
        r.raise_for_status() #非失常返回爬没同常
        r.encoding=r.apparent_encoding#设置返回内容的字符散编码
        return r.text
    except Exception:
        print("download page error")

#网页解析
def parse_html(html):
    data=etree.HTML(html)
    title=data.xpath('//div[@id="u一"]/a/text()')
    url=data.xpath('//div[@id="u一"]/a/@href')
    result=dict()
    for i in range(0,len(title)):
        result[title[i]]=url[i]
    return result

if __name__=='__main__':
    url="http://www.百度.com"
    for k,v in parse_html(down_html(url)).items():
        print(k+"-->"+v)

二

import requests
import csv
from lxml import etree

#高载网页
def download_page(url):
    try:
        r=requests.get(url)#收送要求
        r.raise_for_status() #非失常返回爬没同常
        r.encoding=r.apparent_encoding #设置返回内容的字符散编码
        return r.text #返回网页的文原内容
    except Exception:
        pass

#解析网页
def parse_html(html):
    data=etree.HTML(html)
    books=[]
    for book in data.xpath('//*[@id="tag-book"]/div/ul/li'):
        name=book.xpath("div[二]/h四/a/text()")[0].strip() #书名
        author=book.xpath('div[二]/div/span/text()')[0].strip()#做者
        price=book.xpath("div[二]/span/span/text()")[0].strip()#价钱
        details_url=book.xpath("div[二]/h四/a/@href")[0].strip()#详情页天址
        book=[name,author,price,details_url]
        books.append(book)
    return books

#保留数据
def save_data(file,data): #path文件保留途径，item数据列表铃博网
    with open(file,"w+") as f:
        writer = csv.writer(f)
        for row in data:
            writer.writerow(row)

    print("data saved successfully")

if __name__ == '__main__':
    url="https://www.ryjiaoyu.com/tag/details/七"
    save_data("book四.csv",parse_html(download_page(url)))