python制作mysql数据迁移脚本


Posted in Python onJanuary 01, 2019

用python写了个数据迁移脚本,主要是利用从库将大的静态表导出表空间,载导入到目标实例中。

#!/usr/bin/env python3
#-*- coding:utf8 -*-
#author:zhanbin.liu
#!!!!!DB必须同版本
#python3环境  pip3 install pymysql paramiko

import os
#from pathlib import Path
import sys
import pymysql
import paramiko

#每次只能迁移一个DB下的表,到指定DB
#GRANT SELECT, CREATE, RELOAD, ALTER, LOCK TABLES ON *.* TO 'data_migration'@'192.168.%' IDENTIFIED BY 'data_migration@123';
tables='sqlauto_cluster,sqlauto_user'    #以,分割的字符串,如a,b,c
tableList = tables.split(',')
sourceIp = '192.168.1.101'
sourceDataBase = '/data/mysql/3306/data'
sourceDbName = 'inception_web'
sourceDataDir = os.path.join(sourceDataBase,sourceDbName)
desIp = '192.168.1.102'
desDataBase = '/data/mysql/3306/data'
desDbName = 'inception_web'
desDataDir = os.path.join(desDataBase,desDbName)

# for table in tableList:
#   desFile = Path("%s/%s.ibd" %(desDataDir,table))
#   print(desFile)
#   if desFile.is_file():
#     print("ok")
#   else:
#     print("no")

comUser = 'data_migration'
comPwd = 'data_migration@123'
comPort = 3306

client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

def table_judge():
  print("table_judge")
  sourceTableExist = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  desTableExist = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  sourceTables = []
  desTables = []
  cursor_source = sourceTableExist.cursor()
  cursor_des = desTableExist.cursor()

  for table in tableList:
    #print(table)
    cursor_source.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (sourceDbName,table))
    sourceTable_tmp = cursor_source.fetchall()
    cursor_des.execute("select TABLE_NAME from information_schema.TABLES where TABLE_SCHEMA='%s' and TABLE_NAME='%s';" % (desDbName,table))
    desTable_tmp = cursor_des.fetchall()
    #print(desTable_tmp)
    if sourceTable_tmp is ():
      sourceTables.append(table)
    if desTable_tmp is not ():
      desTables.append(desTable_tmp[0][0])
  sourceTableExist.close()
  desTableExist.close()

  s=d=0
  if sourceTables != []:
    print('迁移源不存在将要迁移的表:',sourceIp,sourceDbName, sourceTables,' 请检查')
    s=1
  if desTables != []:
    print('目标库存在将要迁移的表:',desIp,desDbName,desTables,' 请移除')
    d=1
  if s == 1 or d == 1:
    sys.exit()

def data_sync():
  print('data_sync')
  db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  cursor_db_source = db_source.cursor()
  cursor_db_des = db_des.cursor()

  for table in tableList:
    print("正在同步表:",table)
    cursor_db_source.execute("show create table %s;" % (table))
    createTableSQL = cursor_db_source.fetchall()[0][1]
    print(createTableSQL)
    try:
      cursor_db_des.execute(createTableSQL)
    except Exception as error:
      print(error)
    cursor_db_source.execute("flush table %s with read lock;" % (table))
    cursor_db_des.execute("alter table %s discard tablespace;" % (table))

    client.connect(sourceIp, 22, 'root')
    stdin1, stdout1, stderr1 = client.exec_command("scp %s %s:%s " % (sourceDataDir+"/"+table+".ibd", desIp, desDataDir))
    stdin2, stdout2, stderr2 = client.exec_command("scp %s %s:%s " % (sourceDataDir+"/"+table+".cfg", desIp, desDataDir))
    a_e_1 = stderr1.readlines()
    a_e_2 = stderr2.readlines()
    if a_e_1 != [] or a_e_2 != []:
      print(a_e_1,a_e_2)
      sys.exit()
    client.close()

    client.connect(desIp, 22, 'root')
    stdin3, stdout3, stderr3 = client.exec_command("chown -R mysql.mysql %s*" % (desDataDir+"/"+table))
    a_e_3 = stderr3.readlines()
    if a_e_3 != []:
      print(a_e_1, a_e_2)
      sys.exit()
    client.close()
    #cursor_db_source.execute("select sleep(10);")
    cursor_db_source.execute("unlock tables;")
    cursor_db_des.execute("alter table %s import tablespace;" % (table))
    print("同步完成")

  cursor_db_source.close()
  cursor_db_des.close()

def data_checksum():
  print('data_checksum')
  db_source = pymysql.connect(sourceIp,comUser,comPwd,sourceDbName,comPort,charset='utf8')
  db_des = pymysql.connect(desIp,comUser,comPwd,desDbName,comPort,charset='utf8')
  cursor_db_source = db_source.cursor()
  cursor_db_des = db_des.cursor()

  for table in tableList:
    print("正在校验表:", table)
    cursor_db_source.execute("checksum table %s;" % (table))
    ck_s = cursor_db_source.fetchall()[0][1]
    cursor_db_des.execute("checksum table %s;" % (table))
    ck_d = cursor_db_des.fetchall()[0][1]
    if ck_s != ck_d:
      print("表不一致:",table)
    else:
      print("表一致:",table)

  cursor_db_source.close()
  cursor_db_des.close()

if __name__ == "__main__":
  table_judge()
  data_sync()
  data_checksum()
  print('haha')
Python 相关文章推荐
Python中使用PDB库调试程序
Apr 05 Python
Python中Iterator迭代器的使用杂谈
Jun 20 Python
python 文件操作删除某行的实例
Sep 04 Python
详解Tensorflow数据读取有三种方式(next_batch)
Feb 01 Python
遗传算法python版
Mar 19 Python
python实现黑客字幕雨效果
Jun 21 Python
python机器学习之神经网络实现
Oct 13 Python
Python GUI编程 文本弹窗的实例
Jun 11 Python
Tensorflow设置显存自适应,显存比例的操作
Feb 03 Python
python+django+selenium搭建简易自动化测试
Aug 19 Python
PyCharm2020最新激活码+激活码补丁(亲测最新版PyCharm2020.2激活成功)
Nov 25 Python
Python jiaba库的使用详解
Nov 23 Python
在python中将字符串转为json对象并取值的方法
Dec 31 #Python
对python中Json与object转化的方法详解
Dec 31 #Python
python使用zip将list转为json的方法
Dec 31 #Python
python 获取utc时间转化为本地时间的方法
Dec 31 #Python
python 实现UTC时间加减的方法
Dec 31 #Python
Python从单元素字典中获取key和value的实例
Dec 31 #Python
对Python 两大环境管理神器 pyenv 和 virtualenv详解
Dec 31 #Python
You might like
session 的生命周期是多长
2006/10/09 PHP
PHP系统流量分析的程序
2006/10/09 PHP
PHP制作3D扇形统计图以及对图片进行缩放操作实例
2014/10/23 PHP
php禁止浏览器使用缓存页面的方法
2014/11/07 PHP
PHP7扩展开发之hello word实现方法详解
2018/01/15 PHP
修改js Calendar日历控件 兼容IE9/谷歌/火狐
2013/01/04 Javascript
jQuery Form 页面表单提交的小例子
2013/11/15 Javascript
javascript实例--教你实现扑克牌洗牌功能
2014/05/15 Javascript
浅谈JavaScript字符集
2014/05/22 Javascript
原生js封装二级城市下拉列表的实现代码
2016/06/16 Javascript
移动端界面的适配
2017/01/11 Javascript
vue-cli+webpack记事本项目创建
2017/04/01 Javascript
JS与CSS3实现图片响应鼠标移动放大效果示例
2018/05/04 Javascript
解决vue热替换失效的根本原因
2018/09/19 Javascript
layui导出所有数据的例子
2019/09/10 Javascript
微信小程序学习总结(五)常见问题实例小结
2020/06/04 Javascript
Python调用C/C++动态链接库的方法详解
2014/07/22 Python
Python制作豆瓣图片的爬虫
2017/12/28 Python
Python操作MongoDB数据库的方法示例
2018/01/04 Python
python3+pyqt5+itchat微信定时发送消息的方法
2019/02/20 Python
Pandas中resample方法详解
2019/07/02 Python
Python pandas自定义函数的使用方法示例
2019/11/20 Python
python实现超市商品销售管理系统
2019/11/22 Python
在python3中使用shuffle函数要注意的地方
2020/02/28 Python
Python IDE环境之 新版Pycharm安装详细教程
2020/03/05 Python
Pythonic版二分查找实现过程原理解析
2020/08/11 Python
Kangol帽子官网:坎戈尔袋鼠
2018/09/26 全球购物
金山毒霸系列的笔试题
2013/04/13 面试题
中学生班主任评语
2014/01/30 职场文书
工会换届选举方案
2014/05/21 职场文书
年终晚会活动方案
2014/08/21 职场文书
2014年项目经理工作总结
2014/11/24 职场文书
论文答辩开场白大全
2015/05/27 职场文书
js基础语法与maven项目配置教程案例
2021/07/15 Javascript
抖音动画片,皮皮虾,《治愈系》动画在用这首REMIX作为背景音乐,Anak ,The last world with you完整版
2022/03/16 杂记
SpringBoot集成MongoDB实现文件上传的步骤
2022/04/18 MongoDB