一、簡介
pgloader是一款開源軟件,可以將各種來源的數(shù)據(jù)加載到PostgreSQL數(shù)據(jù)庫中，支持動態(tài)讀取數(shù)據(jù)，使用 COPY 流式傳輸協(xié)議將數(shù)據(jù)加載到 PostgreSQL 數(shù)據(jù)庫中，并使用單獨的線程讀取和寫入數(shù)據(jù)，由于能夠直接從源數(shù)據(jù)庫加載數(shù)據(jù)。今天我們就借助pgloader這款工具實現(xiàn)將MySQL數(shù)據(jù)遷移到PostgreSQL數(shù)據(jù)庫。
二、版本說明
MySQL版本為8.0.31、PostgreSQL版本為13.5。
三、安裝PostgreSQL數(shù)據(jù)庫
1.創(chuàng)建pg用戶及安裝目錄
useradd?postgres??
mkdir?-p?/home/postgresql
2.安裝pg數(shù)據(jù)庫依賴包
yum install -y perl-ExtUtils-Embed readline-devel python3 python3-devel gcc-c++ cmake libarchive openssl-devel
3.pg安裝部署
wget https://ftp.postgresql.org/pub/source/v13.5/postgresql-13.5.tar.gz
tar -zxvf postgresql-13.5.tar.gz? && cd postgresql-13.5/
./configure --prefix=/home/postgresql --with-python --with-perl --with-openssl
make &&make install
/home/postgresql/bin/pg_ctl --version?? ##查看安裝版本
pg_ctl (PostgreSQL) 13.5
4.設(shè)置環(huán)境變量
mkdir -p /app/postgresql/pgdata ##創(chuàng)建數(shù)據(jù)庫存儲目錄
cat >> /etc/profile << EOF
## postgres ##
export PATH=/app/postgresql/bin:$PATH
export LD_LIBRARY_PATH=/app/postgresql/lib
export PGDATA=/app/postgresql/pgdata
EOF
source /etc/profile? #加載環(huán)境變量
pg_ctl --version
pg_ctl (PostgreSQL) 13.5
5.授權(quán)
chown -R postgres.postgres /home/postgresql
su - postgres
initdb? ##初始化
6.創(chuàng)建數(shù)據(jù)庫
pg_ctl start -D $PGDATA? ##啟動數(shù)據(jù)庫
createuser -P zabbix? ##輸出兩次密碼
Enter password for new role:
Enter it again:
createdb -O zabbix -E Unicode -T template0 zabbix? ##創(chuàng)建數(shù)據(jù)庫
四、安裝pgloader工具
1.安裝部署
wget https://codeload.github.com/dimitri/pgloader/tar.gz/refs/tags/v3.6.9
tar -zxvf v3.6.9
dnf -y install freetds-devel? ##安裝依賴包
cd pgloader-3.6.9/
chmod 755 bootstrap-centos.sh
./bootstrap-centos.sh
make pgloader
cp build/bin/pgloader /usr/local/bin/
pgloader --version
pgloader version "3.6.7~devel"
compiled with SBCL 2.2.10-1.rhel8
2.修改Mysql身份認(rèn)證
echo "default-authentication-plugin=mysql_native_password" >> /etc/my.conf ##pgloader不支持caching_sha2_password身份驗證插件,而這個是 MySQL 8 的默認(rèn)設(shè)置，需要修改配置，如MySQL8.0前版本無需操作
systemctl restart mysqld? ##重啟數(shù)據(jù)庫
3.導(dǎo)入Zabbix表結(jié)構(gòu)
vi database/postgresql/schema.sql? ##從INSERT INTO dbversion開始往下全部刪除，參考命令? :.,$d
CREATE INDEX sla_service_tag_1 ON sla_service_tag (slaid);
CREATE TABLE dbversion (
??????? dbversionid????????????? bigint??????????????????????????????????? NOT NULL,
??????? mandatory??????????????? integer???????? DEFAULT '0'?????????????? NOT NULL,
??????? optional???????????????? integer???????? DEFAULT '0'?????????????? NOT NULL,
??????? PRIMARY KEY (dbversionid)
);
INSERT INTO dbversion VALUES ('1','6000000','6000017');
create or replace function hosts_name_upper_upper()
returns trigger language plpgsql as $func$
begin
update hosts set name_upper=upper(name)
where hostid=new.hostid;
psql -Uzabbix -dzabbix -f database/postgresql/schema.sql
4.遷移Mysql配置信息
參考文獻:https://pgloader.readthedocs.io/en/latest/ref/mysql.html
mkdir -p /root/migration && cd /root/migration
vi config.pgloader? ##當(dāng)復(fù)制下面配置的時候請去除所有的注釋
LOAD DATABASE
FROM mysql://zabbix:*****@127.0.0.1:3306/zabbix
INTO postgresql://zabbix:*****@127.0.0.1:5432/zabbix
WITH include no drop,
#當(dāng)列出此選項時，pgloader在加載數(shù)據(jù)時將不包含任何DROP語句。
truncate,
#當(dāng)列出這個選項時，pgloader在將數(shù)據(jù)加載到每個PostgreSQL表之前，對每個PostgreSQL表發(fā)出TRUNCATE命令。刪除表中的所有行，但表結(jié)構(gòu)及其列、約束、索引等保持不變。新行標(biāo)識所用的計數(shù)值重置為該列的種子
create no tables,
#當(dāng)列出此選項時，pgloader在加載數(shù)據(jù)之前跳過表的創(chuàng)建，目標(biāo)表必須已存在。
#此外，當(dāng)使用不創(chuàng)建表時，pgloader從當(dāng)前目標(biāo)數(shù)據(jù)庫獲取元數(shù)據(jù)并檢查類型轉(zhuǎn)換，然后在加載數(shù)據(jù)之前刪除約束和索引，并在加載完成后重新安裝它們。
create no indexes,
#當(dāng)列出此選項時，pgloader將跳過創(chuàng)建索引。
no foreign keys,
#當(dāng)列出此選項時，pgloader將跳過創(chuàng)建外鍵。
reset sequences,
#當(dāng)列出這個選項時，在數(shù)據(jù)加載結(jié)束時，在所有索引都創(chuàng)建完成之后，pgloader將創(chuàng)建的所有PostgreSQL序列重置為它們所附列的當(dāng)前最大值。
data only
#當(dāng)列出此選項時，pgloader只發(fā)出COPY語句，而不進行任何處理。
SET maintenance_work_mem TO '1024MB', work_mem to '512MB'
#設(shè)置maintenance_work_mem和work_mem,根據(jù)自己機器的配置來設(shè)置，越大遷移越快
EXCLUDING TABLE NAMES MATCHING ~/history.*/, ~/trend.*/
#不遷移history表和trends表
ALTER SCHEMA 'zabbix' RENAME TO 'public';
#將pgloader轉(zhuǎn)換生成的zabbix模式更名為public
pgloader config.pgloader ##開始遷移所有的配置不包含歷史數(shù)據(jù)
Total import time????????? ????? 126602??? 12.5 MB????????? 3.820s? ##新庫耗時時間較短
5.查看遷移數(shù)據(jù)
#? psql -Uzabbix -dzabbix -h127.0.0.1
> \d items
????? Column????? |????????? Type?????????? | Collation | Nullable |????????? Default???????? ?
------------------+-------------------------+-----------+----------+---------------------------
?itemid?????????? | bigint????????????????? |?????????? | not null |
?type???????????? | integer???????????????? |?????????? | not null | 0
?snmp_oid???????? | character varying(512)? |?????????? | not null | ''::character varying
?hostid?????????? | bigint????????????????? |?????????? | not null |
?name???????????? | character varying(255)? |?????????? | not null | ''::character varying
?...
?Indexes:
??? "items_pkey" PRIMARY KEY, btree (itemid)
??? "items_1" btree (hostid, key_)
??? "items_3" btree (status)
??? "items_4" btree (templateid)
??? "items_5" btree (valuemapid)
??? "items_6" btree (interfaceid)
??? "items_7" btree (master_itemid)
??? "items_8" btree (key_)
??? "items_9" btree (hostid, name_upper)
遷移MySQL歷史數(shù)據(jù)
# cd /root/migration
# vi data.pgloader? ##過濾掉history和trends七張表，每個大版本的表數(shù)量不相同，下面過濾的表請按實際版本中表數(shù)量過濾
LOAD DATABASE
FROM mysql://zabbix:*****@127.0.0.1:3306/zabbix
INTO postgresql://zabbix:*****@127.0.0.1:5432/zabbix
WITH include no drop,
no truncate,
create no tables,
create no indexes,
no foreign keys,
reset sequences,
data only
SET maintenance_work_mem TO '1024MB', work_mem TO '512MB'
EXCLUDING TABLE NAMES MATCHING 'acknowledges',
'actions',
'alerts',
'auditlog',
'autoreg_host',
'conditions',
'config',
'config_autoreg_tls',
'corr_condition',
'corr_condition_group',
'corr_condition_tag',
'corr_condition_tagpair',
'corr_condition_tagvalue',
'corr_operation',
'correlation',
'dashboard',
'dashboard_page',
'dashboard_user',
'dashboard_usrgrp',
'dbversion',
'dchecks',
'dhosts',
'drules',
'dservices',
'escalations',
'event_recovery',
'event_suppress',
'event_tag',
'events',
'expressions',
'functions',
'globalmacro',
'globalvars',
'graph_discovery',
'graph_theme',
'graphs',
'graphs_items',
'group_discovery',
'group_prototype',
'ha_node',
'host_discovery',
'host_inventory',
'host_tag',
'hostmacro',
'hosts',
'hosts_groups',
'hosts_templates',
'housekeeper',
'hstgrp',
'httpstep',
'httpstep_field',
'httpstepitem',
'httptest',
'httptest_field',
'httptest_tag',
'httptestitem',
'icon_map',
'icon_mapping',
'ids',
'images',
'interface',
'interface_discovery',
'interface_snmp',
'item_condition',
'item_discovery',
'item_parameter',
'item_preproc',
'item_rtdata',
'item_tag',
'items',
'lld_macro_path',
'lld_override',
'lld_override_condition',
'lld_override_opdiscover',
'lld_override_operation',
'lld_override_ophistory',
'lld_override_opinventory',
'lld_override_opperiod',
'lld_override_opseverity',
'lld_override_opstatus',
'lld_override_optag',
'lld_override_optemplate',
'lld_override_optrends',
'maintenance_tag',
'maintenances',
'maintenances_groups',
'maintenances_hosts',
'maintenances_windows',
'media',
'media_type',
'media_type_message',
'media_type_param',
'module',
'opcommand',
'opcommand_grp',
'opcommand_hst',
'opconditions',
'operations',
'opgroup',
'opinventory',
'opmessage',
'opmessage_grp',
'opmessage_usr',
'optemplate',
'problem',
'problem_tag',
'profiles',
'proxy_autoreg_host',
'proxy_dhistory',
'proxy_history',
'regexps',
'report',
'report_param',
'report_user',
'report_usrgrp',
'rights',
'role',
'role_rule',
'script_param',
'scripts',
'service_alarms',
'service_problem',
'service_problem_tag',
'service_status_rule',
'service_tag',
'services',
'services_links',
'sessions',
'sla',
'sla_excluded_downtime',
'sla_schedule',
'sla_service_tag',
'sysmap_element_trigger',
'sysmap_element_url',
'sysmap_shape',
'sysmap_url',
'sysmap_user',
'sysmap_usrgrp',
'sysmaps',
'sysmaps_element_tag',
'sysmaps_elements',
'sysmaps_link_triggers',
'sysmaps_links',
'tag_filter',
'task',
'task_acknowledge',
'task_check_now',
'task_close_problem',
'task_data',
'task_remote_command',
'task_remote_command_result',
'task_result',
'timeperiods',
'token',
'trigger_depends',
'trigger_discovery',
'trigger_queue',
'trigger_tag',
'triggers',
'users',
'users_groups',
'usrgrp',
'valuemap',
'valuemap_mapping',
'widget',
'widget_field'
ALTER SCHEMA 'zabbix' RENAME TO 'public';
# pgloader data.pgloader? ##只有一臺Server監(jiān)控數(shù)據(jù)用時較短
2023-06-11T21:00:32.007800+21:00 LOG pgloader version "3.6.7~devel"
2023-06-11T21:00:32.007800+21:00 LOG Migrating from #<MYSQL-CONNECTION mysql://zabbix@127.0.0.1:3306/zabbix {10067B4643}>
2023-06-11T21:00:32.007800+21:00 LOG Migrating into #<PGSQL-CONNECTION pgsql://zabbix@127.0.0.1:5432/zabbix {10067B47D3}>
2023-06-11T21:00:32.007800+21:00 WARNING Source column "public"."history_uint"."value" is casted to type "bigint" which is not the same as "numeric", the type of current target database column "public"."history_uint"."value".
2023-06-11T21:00:32.007800+21:00 WARNING Source column "public"."trends_uint"."value_min" is casted to type "bigint" which is not the same as "numeric", the type of current target database column "public"."trends_uint"."value_min".
2023-06-11T21:00:32.007800+21:00 WARNING Source column "public"."trends_uint"."value_avg" is casted to type "bigint" which is not the same as "numeric", the type of current target database column "public"."trends_uint"."value_avg".
2023-06-11T21:00:32.007800+21:00 WARNING Source column "public"."trends_uint"."value_max" is casted to type "bigint" which is not the same as "numeric", the type of current target database column "public"."trends_uint"."value_max".
2023-06-11T21:00:32.007800+21:00 LOG report summary reset
???????????? table name???? errors?????? rows????? bytes????? total time
-----------------------? ---------? ---------? ---------? --------------
??????? fetch meta data????????? 0????????? 7???????????????????? 0.076s
-----------------------? ---------? ---------? ---------? --------------
???????? public.history????????? 0?????? 8008?? 286.8 kB????????? 0.110s
??? public.history_uint????????? 0?????? 2429??? 74.1 kB????????? 0.067s
???? public.trends_uint????????? 0???????? 50???? 1.6 kB????????? 0.082s
??? public.history_text????????? 0????????? 2??? 52.6 kB????????? 0.117s
????????? public.trends????????? 0??????? 142???? 8.4 kB????????? 0.020s
???? public.history_str????????? 0???????? 12???? 0.9 kB????????? 0.036s
???? public.history_log????????? 0????????? 0???????????????????? 0.025s
-----------------------? ---------? ---------? ---------? --------------
COPY Threads Completion????????? 0????????? 4???????????????????? 0.171s
??????? Reset Sequences????????? 0????????? 0???????????????????? 0.027s
?????? Install Comments????????? 0????????? 0???????????????????? 0.000s
-----------------------? ---------? ---------? ---------? --------------
????? Total import time????????? ?????? 10643?? 424.5 kB????????? 0.198s
6.查看歷史數(shù)據(jù)遷移
psql -Uzabbix
\c zabbix
zabbix=> select * from history;
?itemid |?? clock??? |???????? value????????? |??? ns??? ?
--------+------------+------------------------+-----------
? 10073 | 1670483513 |???? 0.8324606300857088 | 661499763
? 10073 | 1670483573 |???? 1.0157088072329086 | 718082482
? 10073 | 1670483693 |???? 0.8991120422218923 | 907381983
? 10073 | 1670483753 |????? 1.015714184025525 | 963646786
? 10073 | 1670483813 |???? 1.0329172154884476 |? 19686404
? 10073 | 1670483873 |???? 1.0158031677606336 |? 70690315
? 10073 | 1670483933 |???? 1.0157542389272938 | 124586880
? 10073 | 1670483993 |????? 1.015691219317195 | 182209551
? 10073 | 1670484053 |???? 1.0156428524089065 | 242692284
7.設(shè)置主外鍵
cat schema.sql |tail -n +2090 > altertable.sql > altertable.sql? ##將所有ALTER以及另一段sql放入altertable.sql中
psql -Uzabbix -dzabbix -f database/postgresql/altertable.sql
psql -Uzabbix
\c zabbix
\d+ items
...
Foreign-key constraints:
??? "c_items_1" FOREIGN KEY (hostid) REFERENCES hosts(hostid) ON DELETE CASCADE
??? "c_items_2" FOREIGN KEY (templateid) REFERENCES items(itemid) ON DELETE CASCADE
??? "c_items_3" FOREIGN KEY (valuemapid) REFERENCES valuemap(valuemapid)
??? "c_items_4" FOREIGN KEY (interfaceid) REFERENCES interface(interfaceid)
??? "c_items_5" FOREIGN KEY (master_itemid) REFERENCES items(itemid) ON DELETE CASCADE
五、Zabbix前端訪問測試
1.數(shù)據(jù)庫連接配置
2.測試連接成功
注:按照實際環(huán)境測試環(huán)境先熟悉驗證，不建議直接生產(chǎn)操作該步驟
博客可能不能及時回復(fù)問題，技術(shù)問題歡迎加入交流。
具有豐富的模板資源及模板開發(fā)能力、項目落地管理經(jīng)驗分享歡迎加入交流文章來源：http://www.zghlxwxcb.cn/news/detail-648023.html
微信號:king_songax文章來源地址http://www.zghlxwxcb.cn/news/detail-648023.html
到了這里，關(guān)于利用pgloader工具將MySQL數(shù)據(jù)遷移至PostgreSQL數(shù)據(jù)庫的文章就介紹完了。如果您還想了解更多內(nèi)容，請在右上角搜索TOY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！
