霍比特人小說爬取，使用 selenium 模塊調(diào)用谷歌瀏覽器，無界面模式爬取小說網(wǎng)站信息，將數(shù)據(jù)按照每次2000字符在mysql中保存。
文章來源地址http://www.zghlxwxcb.cn/news/detail-733278.html

# https://www.shukuai9.com/b/324694/
# 導(dǎo)入需要的庫(kù)
from selenium import webdriver
# 導(dǎo)入Keys模塊，用于模擬鍵盤按鍵操作
from selenium.webdriver.common.keys import Keys
#數(shù)據(jù)庫(kù)
import pymysql

#獲取文本，拆分為4000字每段
def text_sql(title,dimension,text):
    text = text.replace(chr(9),'').replace(chr(10),'').replace(chr(13),'')
    print(title,dimension,len(text),len(text)//2000)
    for i in range((len(text)//2000)+1):
        str_num = i*2000
        end_num = (i+1)*2000
        #print(str_num,end_num)
        #print(i,text[str_num:end_num])
        try:
            start = mysql_in(title,dimension,i,text[str_num:end_num])
        except Exception as e:
            print(e)
            start='異常'
        print(title,dimension,i,start)

#數(shù)據(jù)庫(kù)登錄
def mysql_execute(in_sql, leixing):
    # 登錄數(shù)據(jù)庫(kù)
    conn = pymysql.connect(host='127.0.0.1', port=3306, user='szc_sql', password='szcNSP850219', database='szc_sql',charset='utf8')
    # # 登錄數(shù)據(jù)庫(kù)
    # dsn = "134.80.200.216/xxx"
    # try:
    #     conn = cx_Oracle.connect(user="zbweb", password="zibo_xxx", dsn=dsn, encoding="UTF-8")
    # except:
    #     time.sleep(10)
    #     conn = cx_Oracle.connect(user="zbweb", password="zibo_xxx", dsn=dsn, encoding="UTF-8")

    # 得到一個(gè)可以執(zhí)行SQL語句的光標(biāo)對(duì)象
    cursor = conn.cursor()
    # 數(shù)據(jù)庫(kù)執(zhí)行導(dǎo)入的語句
    if leixing == '數(shù)量':
        # 反饋數(shù)量
        count = cursor.execute(in_sql)
    elif leixing == '單條':
        # 反饋單條
        cursor.execute(in_sql)
        count = cursor.fetchone()[0]
    elif leixing == '多條':
        # 反饋多條
        cursor.execute(in_sql)
        count = cursor.fetchall()
    elif leixing == '編輯':
        count = cursor.execute(in_sql)
        conn.commit()
    # 關(guān)閉光標(biāo)對(duì)象
    cursor.close()
    # 關(guān)閉數(shù)據(jù)庫(kù)連接
    conn.close()
    # 反饋
    return count

def mysql_in(title,dimension,num,text):
    print(num,type(num))
    sql="delete from novel_text where title='%s' and dimension='%s' and num='%s' " % (title,dimension,str(num))
    mysql_execute(sql, '編輯')
    sql = "insert into novel_text (title,dimension,num,text) values('%s','%s','%s','%s') " % (title,dimension,str(num),text)
    print('sql',sql)
    mysql_execute(sql, '編輯')
    return '完成'


# 創(chuàng)建Chrome瀏覽器對(duì)象
chrome_opt = webdriver.ChromeOptions()
# 開啟無界面模式
chrome_opt.add_argument('--headless')
# 禁用gpu
chrome_opt.add_argument('--disable-gpu')
# 創(chuàng)建Chrome瀏覽器實(shí)例
driver = webdriver.Chrome(executable_path='./chromedriver.exe',options=chrome_opt)
# 打開網(wǎng)頁(yè)
driver.get("https://www.shukuai9.com/b/324694/")
# 等待頁(yè)面加載完成
driver.implicitly_wait(10)
# 獲取搜索結(jié)果列表元素
search_results = driver.find_elements_by_css_selector("dd")
search_title = driver.find_elements_by_css_selector("h1")
print('search_title',search_title)

for text in search_title:
    print('text',text.text)
    search_title_text = text.text

hbt_key = {}
# 輸出搜索結(jié)果標(biāo)題和鏈接
for result in search_results:
    title = result.find_element_by_css_selector("a").text
    link = result.find_element_by_css_selector("a").get_attribute("href")
    num = str(title).index('節(jié)')
    title_num = title[1:num]
    #print(title_num,biaoti, link)
    hbt_key[title_num]=[title,link]

#print('hbt_key',hbt_key)

#獲取鍵值對(duì)個(gè)數(shù)
for i in range(len(hbt_key)):
    print(hbt_key[str(i+1)])

    title = hbt_key[str(i + 1)][0]
    link = hbt_key[str(i + 1)][1]
    # 打開網(wǎng)頁(yè)
    driver.get(link)
    # 等待頁(yè)面加載完成
    driver.implicitly_wait(10)
    # 獲取搜索結(jié)果列表元素
    search_results = driver.find_element_by_xpath('//*[@id="content"]').text
    #print(search_results)
    # 輸出搜索結(jié)果標(biāo)題和鏈接
    search_text = ''
    for t in search_results:
        search_text += t
    print(len(search_text),type(search_text))
    #文字留存
    text_sql(search_title_text,title,search_text)
    #break

到了這里，關(guān)于20230908_python練習(xí)_selenium模塊爬取網(wǎng)頁(yè)小說練習(xí)的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！