基于Hive的天氣情況大數(shù)據(jù)分析系統(tǒng)（通過hive進行大數(shù)據(jù)分析將分析的數(shù)據(jù)通過sqoop導(dǎo)入到mysql，通過Django基于mysql的數(shù)據(jù)做可視化）

1. Hive介紹：
   Hive是建立在Hadoop之上的數(shù)據(jù)倉庫基礎(chǔ)架構(gòu)，它提供了類似于SQL的語言（HQL），可以對大規(guī)模數(shù)據(jù)集進行查詢和分析。通過Hive，我們可以在分布式存儲系統(tǒng)中進行復(fù)雜的數(shù)據(jù)處理和分析。

2. Sqoop簡介：
   Sqoop是一個用于在Apache Hadoop和關(guān)系型數(shù)據(jù)庫之間傳輸數(shù)據(jù)的工具。我們可以使用Sqoop將Hive中的分析結(jié)果導(dǎo)出到關(guān)系型數(shù)據(jù)庫中，如MySQL，以便進一步處理和可視化。

3. Django概述：
   Django是一個高級的Python Web框架，它提供了一系列工具和庫，用于快速構(gòu)建Web應(yīng)用程序。我們可以利用Django連接到MySQL數(shù)據(jù)庫，處理數(shù)據(jù)，并將其呈現(xiàn)為可視化界面。

Hive大數(shù)據(jù)分析sql，基于數(shù)據(jù)創(chuàng)建hive表，然后進行數(shù)據(jù)分析

-- 創(chuàng)建數(shù)據(jù)庫
CREATE DATABASE IF NOT EXISTS big_data;

-- 切換到big_data數(shù)據(jù)庫
USE big_data;

 load data local inpath '/export/server/28' INTO TABLE weather_data;
-- 創(chuàng)建weather_data表
CREATE TABLE IF NOT EXISTS weather_data (
    `date` STRING,
    high_temperature STRING,
    low_temperature STRING,
    weather STRING,
    wind_direction STRING,
    city STRING
)ROW FORMAT DELIMITED
FIELDS TERMINATED BY ',';

-- 插入數(shù)據(jù)到weather_data表（示例數(shù)據(jù)）
INSERT INTO TABLE weather_data VALUES
(1, '2022-01-01 周六', '6°', '-7°', '晴', '西北風(fēng)3級', '北京'),
(2, '2022-01-02 周日', '2°', '-7°', '多云', '南風(fēng)2級', '北京');

-- 創(chuàng)建etl_weather_data表
CREATE TABLE IF NOT EXISTS etl_weather_data (
    `date` STRING,
    day_of_week STRING,
    high_temperature INT,
    low_temperature INT,
    weather STRING,
    wind_direction STRING,
    wind_speed STRING,
    city STRING
);

-- 插入數(shù)據(jù)到etl_weather_data表
INSERT INTO TABLE etl_weather_data
SELECT
    SUBSTR(`date`, 1, INSTR(`date`, ' ') - 1) AS `date`,
    SUBSTR(`date`, INSTR(`date`, ' ') + 1) AS day_of_week,
    CAST(SUBSTR(high_temperature, 1, INSTR(high_temperature, '°') - 1) AS INT) AS high_temperature,
    CAST(SUBSTR(low_temperature, 1, INSTR(low_temperature, '°') - 1) AS INT) AS low_temperature,
    weather,
    REGEXP_REPLACE(SUBSTR(wind_direction, 1, INSTR(wind_direction, '級') - 1), '[0-9]', '') AS wind_direction,
    SUBSTR(SUBSTR(wind_direction, INSTR(wind_direction, '風(fēng)') + 1),1,1) AS wind_speed,
    city
FROM
    weather_data;

-- 1.統(tǒng)計一年中每個城市晴天個數(shù)的top10
CREATE TABLE IF NOT EXISTS top_sunny_cities (
    city STRING,
    sunny_days_count INT
);

INSERT INTO TABLE top_sunny_cities
SELECT
    city,
    COUNT(*) AS sunny_days_count
FROM
    etl_weather_data
WHERE
    weather LIKE '%晴%'
GROUP BY
    city
ORDER BY
    sunny_days_count DESC
LIMIT 10;


-- 2.統(tǒng)計北京一年中每個月的溫差變化
CREATE TABLE IF NOT EXISTS monthly_max_temperature_difference (
    month_year STRING,
    max_temperature_difference INT
);

INSERT INTO TABLE monthly_max_temperature_difference
SELECT
    CONCAT(YEAR(`date`), '-', LPAD(MONTH(`date`), 2, '0')) AS month_year,
    MAX(high_temperature - low_temperature) AS max_temperature_difference
FROM
    etl_weather_data
WHERE
    city = '北京'
GROUP BY
    YEAR(`date`), MONTH(`date`);

-- 3.統(tǒng)計城市出現(xiàn)3級以上風(fēng)速最多的10個城市
CREATE TABLE IF NOT EXISTS top_cities_high_wind (
    city STRING,
    high_wind_days_count INT
);

INSERT INTO TABLE top_cities_high_wind
SELECT
    city,
    COUNT(*) AS high_wind_days_count
FROM
    etl_weather_data
WHERE
    CAST(wind_speed AS INT) >= 3
GROUP BY
    city
ORDER BY
    high_wind_days_count DESC
LIMIT 10;

基于sqoop將數(shù)據(jù)導(dǎo)入到mysql中

sqoop export \
  --connect jdbc:mysql://192.168.138.1:3306/big_data \
  --username root --password '123456' \
  --table top_sunny_cities_sqoop \
  --export-dir /hive/warehouse/big_data.db/big_data.dbbig_data.db/top_sunny_cities \
  --input-fields-terminated-by '\001' \
  --input-lines-terminated-by '\n';

sqoop export \
  --connect jdbc:mysql:// 192.168.138.1:3306/big_data \
  --username root --password 123456 \
  --table monthly_max_temperature_difference \
  --export-dir /user/hive/warehouse/big_data.db/big_data.dbmonthly_max_temperature_difference \
  --input-fields-terminated-by '\001' \
  --input-lines-terminated-by '\n'

sqoop export \
  --connect jdbc:mysql:// 192.168.138.1:3306/big_data \
  --username root --password 123456 \
  --table top_cities_high_wind \
  --export-dir /user/hive/warehouse/big_data.db/big_data.dbtop_cities_high_wind \
  --input-fields-terminated-by '\001' \
  --input-lines-terminated-by '\n'

基于mysql數(shù)據(jù)使用Django做數(shù)據(jù)可視化

from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
from django.shortcuts import render, redirect
from django.shortcuts import render
from pyecharts import options as opts
from pyecharts.charts import Line, Bar, Pie, Grid
# Create your views here.


from django.shortcuts import render
from pyecharts.globals import ThemeType

from api.service.task_service import get_user, top_sunny_cities, monthly_max_temperature_difference, \
    top_cities_high_wind, top_rainy_cities, monthly_rainy_days, yearly_min_temperatures, daily_wind_speed, \
    daily_temperature_difference, register_user


def login_page(request):
    if request.method == 'POST':
        username = request.POST.get('username')
        password = request.POST.get('password')
        user = get_user(username,password)
        if user is not None:
            return redirect('home')
        else:
            return render(request, 'login.html', {'error_message': 'Invalid login credentials.'})

    return render(request, 'login.html')

def register_view(request):
    # 處理注冊邏輯
    if request.method == 'GET':
        username = request.GET.get('username')
        password = request.GET.get('password')
        if username and password:
            register_user(username,password)
            return HttpResponse("注冊成功！")
        return render(request, 'register.html')  # 使用你的注冊模板路徑

def home(request):
    print(2)
    return render(request, 'home.html')

def data_analysis(request, button_id):
    return render(request, 'data_analysis.html', {'button_id': button_id})

def data_analysis(request, button_id):


    # 根據(jù)按鈕 ID 進行不同的處理
    if button_id == 1:
        x,y = top_sunny_cities()
        line_chart = (
            Line()
            .add_xaxis(xaxis_data=x)
            .add_yaxis(series_name="晴天個數(shù)", y_axis=y)
            .set_global_opts(title_opts=opts.TitleOpts(title="一年中每個城市晴天個數(shù)的top10"))
        )
        chart_html = line_chart.render_embed()
        button_name = "折線圖"
    elif button_id == 2:
        x,y = monthly_max_temperature_difference()
        line_chart = (
            Line()
            .add_xaxis(xaxis_data=x)
            .add_yaxis(series_name="溫差值", y_axis=y)
            .set_global_opts(title_opts=opts.TitleOpts(title="北京一年中每個月的溫差變化"))
        )
        chart_html = line_chart.render_embed()
        button_name = "折線圖"
    elif button_id == 3:
        x,y = top_cities_high_wind()
        bar_chart = (
            Bar()
            .add_xaxis(xaxis_data=x)
            .add_yaxis(series_name="3級風(fēng)速次數(shù)",y_axis=y)
            .set_global_opts(title_opts=opts.TitleOpts(title="出現(xiàn)3級以上風(fēng)速的top10個城市"))
        )
        chart_html = bar_chart.render_embed()
        button_name = "條形圖"
    elif button_id == 4:
        x, y = top_rainy_cities()
        bar_chart = (
            Bar()
            .add_xaxis(xaxis_data=x)
            .add_yaxis(series_name="雨天數(shù)量", y_axis=y)
            .set_global_opts(title_opts=opts.TitleOpts(title="多雨城市的top10"))
        )
        chart_html = bar_chart.render_embed()
        button_name = "條形圖"

    elif button_id == 5:
        x, y = monthly_rainy_days()
        pie = Pie()
        pie.add("", list(zip(x, y)))
        pie.set_global_opts(title_opts={"text": "杭州每月雨天變化", "subtext": "2022年"},
                            legend_opts=opts.LegendOpts(orient="vertical", pos_right="right", pos_top="center"))
        chart_html = pie.render_embed()
        button_name = "餅圖"

    elif button_id == 6:
        x, y = yearly_min_temperatures()
        line_chart = (
            Line()
            .add_xaxis(xaxis_data=x)
            .add_yaxis(series_name="溫度", y_axis=y)
            .set_global_opts(title_opts=opts.TitleOpts(title="城市一年中最低的溫度top10"))
        )
        chart_html = line_chart.render_embed()
        button_name = "折線圖"

    elif button_id == 7:
        x,y=daily_temperature_difference()
        # 創(chuàng)建餅圖
        pie = (
            Pie(init_opts=opts.InitOpts(width="800px", height="600px"))
            .add(
                series_name="南京10月份1～10號溫差變化",
                data_pair=list(zip(x, y)),
                radius=["40%", "75%"],  # 設(shè)置內(nèi)外半徑，實現(xiàn)空心效果
                label_opts=opts.LabelOpts(is_show=True, position="inside"),
            )
            .set_global_opts(title_opts=opts.TitleOpts(title="南京10月份1～10號溫差變化"),
                             legend_opts=opts.LegendOpts(orient="vertical", pos_right="right", pos_top="center"),
                             )
            .set_series_opts(  # 設(shè)置系列選項，調(diào)整 is_show 閾值
                label_opts=opts.LabelOpts(is_show=True)
            )
        )
        chart_html = pie.render_embed()
        button_name = "餅圖"

    elif button_id == 8:
        x,y=daily_wind_speed()

        bar_chart = (
            Bar()
            .add_xaxis(xaxis_data=x)
            .add_yaxis(series_name="風(fēng)速級別", y_axis=y)
            .set_global_opts(title_opts=opts.TitleOpts(title="南京10月份每天的風(fēng)速變化"))
        )
        chart_html = bar_chart.render_embed()
        button_name = "條形圖"

    return render(request, 'data_analysis.html', {'chart_html': chart_html, 'button_name': button_name})

展示Django項目運行結(jié)果







如有遇到問題可以找小編溝通交流哦。另外小編幫忙輔導(dǎo)大課作業(yè)，學(xué)生畢設(shè)等。不限于python，java，大數(shù)據(jù)等。
文章來源地址http://www.zghlxwxcb.cn/news/detail-849671.html

到了這里，關(guān)于基于Hive的天氣情況大數(shù)據(jù)分析系統(tǒng)（通過hive進行大數(shù)據(jù)分析將分析的數(shù)據(jù)通過sqoop導(dǎo)入到mysql，通過Django基于mysql的數(shù)據(jù)做可視化）的文章就介紹完了。如果您還想了解更多內(nèi)容，請在右上角搜索TOY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！