1. 項目簡介

---

在中國天氣網(wǎng)(天氣網(wǎng))中輸入一個城市的名稱，例如輸入深圳，那么會轉(zhuǎn)到地址深圳天氣預(yù)報,深圳7天天氣預(yù)報,深圳15天天氣預(yù)報,深圳天氣查詢的網(wǎng)頁顯示深圳的天氣預(yù)報，其中101280601是深圳的代碼，每個城市或者地區(qū)都有一個代碼。如下圖：

在上圖中可以看到，深圳今天，7天，8-15天等的天氣數(shù)據(jù)，這里爬取7天的天氣預(yù)報數(shù)據(jù)。

2. HTML 代碼分析

---

分析這段代碼：

---

7天的天氣預(yù)報實際上在一個<ul class="t clearfix">元素中，每天是一個M<li>元素，7天的結(jié)構(gòu)差不多是一樣的(注意：今天沒有最高溫度與最低溫度)。

3. 爬取天氣預(yù)報數(shù)據(jù)

---

from bs4 import BeautifulSoup
from bs4.dammit import UnicodeDammit  # BS內(nèi)置庫，用于推測文檔編碼
import urllib.request  # 發(fā)起請求，獲取響應(yīng)

url = "http://www.weather.com.cn/weather/101280601.shtml"

try:
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.78"}
    req = urllib.request.Request(url, headers=headers)  # 創(chuàng)建請求對象
    data = urllib.request.urlopen(req)  # 發(fā)起請求
    data = data.read()  # 獲得響應(yīng)體
    dammit = UnicodeDammit(data, ["utf-8", "gbk"])
    data = dammit.unicode_markup  # 解碼
    soup = BeautifulSoup(data, "lxml")
    lis = soup.select("ul[class='t clearfix'] li")
    x = 0
    for li in lis:
        try:
            date = li.select('h1')[0].text
            weather = li.select('p[class="wea"]')[0].text
            if x == 0:  # 為今天只有一個溫度做判斷 <i>14℃</i>
                x += 1
                temp = li.select('p[class="tem"] i')[0].text
            else:
                temp = li.select('p[class="tem"] span')[0].text + "/" + li.select('p[class="tem"] i')[0].text
            print(date, weather, temp)
            # 22日（今天） 晴 14℃
            # 23日（明天） 晴 23℃/14℃
            # 24日（后天） 晴轉(zhuǎn)多云 25℃/13℃
            # 25日（周六） 多云 21℃/13℃
            # 26日（周日） 多云轉(zhuǎn)晴 22℃/12℃
            # 27日（周一） 晴 21℃/12℃
            # 28日（周二） 晴 24℃/14℃
        except Exception as err:
            print(err)
except Exception as err:
    print(err)

4. 爬取與存儲天氣預(yù)報數(shù)據(jù)

---

獲取北京、上海、廣州、深圳等城市的代碼，爬取這些城市的天氣預(yù)報數(shù)據(jù)，并存儲到sqllite數(shù)據(jù)庫weathers.db中，存儲的數(shù)據(jù)表weathers是：

create table weathers (wCity varchar(16),wDate varchar(16),wWeather varchar(64),wTemp varchar(32),constraint pk_weather primary key (wCity,wDate))"

編寫程序依次爬取各個城市的天氣預(yù)報數(shù)據(jù)存儲在數(shù)據(jù)庫中，程序如下：

from bs4 import BeautifulSoup
from bs4.dammit import UnicodeDammit
import urllib.request
import sqlite3


# 天氣數(shù)據(jù)庫
class WeatherDB:
    def __init__(self):
        self.cursor = None
        self.con = None

    def openDB(self):
        self.con = sqlite3.connect("weathers.db")
        self.cursor = self.con.cursor()
        try:
            self.cursor.execute(
                "create table weathers (wCity varchar(16),"
                "wDate varchar(16),"
                "wWeather varchar(64),"
                "wTemp varchar(32),"
                "constraint pk_weather primary key (wCity,wDate))")
        except Exception as err:
            print(err)
            self.cursor.execute("delete from weathers")

    def closeDB(self):
        self.con.commit()
        self.con.close()

    def insert(self, city, date, weather, temp):
        try:
            self.cursor.execute("insert into weathers (wCity,wDate,wWeather,wTemp) values (?,?,?,?)",
                                (city, date, weather, temp))
        except Exception as err:
            print(err)

    def show(self):
        self.cursor.execute("select * from weathers")
        rows = self.cursor.fetchall()
        print("%-16s%-16s%-32s%-16s" % ("city", "date", "weather", "temp"))
        for row in rows:
            print("%-16s%-16s%-32s%-16s" % (row[0], row[1], row[2], row[3]))


# 天氣預(yù)報
class WeatherForecast:
    def __init__(self):
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) "
                          "Gecko/2008072421 Minefield/3.0.2pre"}
        self.cityCode = {"北京": "101010100", "上海": "101020100", "廣州": "101280101", "深圳": "101280601"}

    def forecastCity(self, city):
        if city not in self.cityCode.keys():
            print(city + " 找不到代碼")
            return
        url = "http://www.weather.com.cn/weather/" + self.cityCode[city] + ".shtml"
        try:
            req = urllib.request.Request(url, headers=self.headers)
            data = urllib.request.urlopen(req)
            data = data.read()
            dammit = UnicodeDammit(data, ["utf-8", "gbk"])
            data = dammit.unicode_markup
            soup = BeautifulSoup(data, "lxml")
            lis = soup.select("ul[class='t clearfix'] li")
            x = 0
            for li in lis:
                try:
                    date = li.select('h1')[0].text
                    weather = li.select('p[class="wea"]')[0].text
                    if x == 0:  # 為今天只有一個溫度做判斷 <i>14℃</i>
                        x += 1
                        temp = li.select('p[class="tem"] i')[0].text
                    else:
                        temp = li.select('p[class="tem"] span')[0].text + "/" + li.select('p[class="tem"] i')[0].text
                    print(city, date, weather, temp)
                    self.db.insert(city, date, weather, temp)
                except Exception as err:
                    print(err)
        except Exception as err:
            print(err)

    def process(self, cities):
        self.db = WeatherDB()
        self.db.openDB()

        for city in cities:
            self.forecastCity(city)

        # self.db.show()
        self.db.closeDB()


ws = WeatherForecast()
ws.process(["北京", "上海", "廣州", "深圳"])
print("completed")

程序執(zhí)行結(jié)果如下：

---

下一篇文章：爬取大學(xué)排名信息文章來源地址http://www.zghlxwxcb.cn/news/detail-458471.html