Merge branch 'master' into num_snaps

Author: davidducos

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 2.6)
 project(mydumper)
-set(VERSION 0.10.0)
+set(VERSION 0.10.6)
 set(ARCHIVE_NAME "${CMAKE_PROJECT_NAME}-${VERSION}")
 
 #Required packages
@@ -32,15 +32,14 @@ CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_SOURCE_DI
 
 
 if (WITH_BINLOG)
-
-  add_executable(mydumper mydumper.c binlog.c server_detect.c g_unix_signal.c connection.c getPassword.c)
+  add_executable(mydumper mydumper.c binlog.c server_detect.c g_unix_signal.c connection.c getPassword.c logging.c set_verbose.c)
 else (WITH_BINLOG)
-  add_executable(mydumper mydumper.c server_detect.c g_unix_signal.c connection.c getPassword.c)
+  add_executable(mydumper mydumper.c server_detect.c g_unix_signal.c connection.c getPassword.c logging.c set_verbose.c)
 endif (WITH_BINLOG)
-target_link_libraries(mydumper ${MYSQL_LIBRARIES} ${GLIB2_LIBRARIES} ${GTHREAD2_LIBRARIES} ${GIO2_LIBRARIES} ${GOBJECT2_LIBRARIES} ${PCRE_PCRE_LIBRARY} ${ZLIB_LIBRARIES} stdc++)
+target_link_libraries(mydumper ${MYSQL_LIBRARIES} ${GLIB2_LIBRARIES} ${GTHREAD2_LIBRARIES} ${GIO2_LIBRARIES} ${GOBJECT2_LIBRARIES} ${PCRE_PCRE_LIBRARY} ${ZLIB_LIBRARIES} stdc++ m)
 
 
-add_executable(myloader myloader.c connection.c getPassword.c)
+add_executable(myloader myloader.c connection.c getPassword.c logging.c set_verbose.c)
 target_link_libraries(myloader ${MYSQL_LIBRARIES} ${GLIB2_LIBRARIES} ${GTHREAD2_LIBRARIES} ${PCRE_PCRE_LIBRARY} ${ZLIB_LIBRARIES} stdc++)
 
 INSTALL(TARGETS mydumper myloader

README.md

@@ -5,21 +5,84 @@
 * Consistency - maintains snapshot across all threads, provides accurate master and slave log positions, etc
 * Manageability - supports PCRE for specifying database and tables inclusions and exclusions
 
+## Dependencies for building mydumper
+
+One needs to install development tools:
+* Ubuntu or Debian: 
+```
+apt-get install cmake g++ git
+```
+* Fedora, RedHat and CentOS:
+```
+yum install -y cmake gcc gcc-c++ git make
+```
+* MacOSX:
+```
+port install pkgconfig cmake
+```
+One needs to install development versions of GLib, ZLib and PCRE:
+* Ubuntu or Debian: 
+```
+apt-get install libglib2.0-dev zlib1g-dev libpcre3-dev libssl-dev
+```
+* Fedora, RedHat and CentOS: 
+```
+yum install -y glib2-devel mysql-devel openssl-devel pcre-devel zlib-devel
+```
+* openSUSE: 
+```
+zypper install glib2-devel libmysqlclient-devel pcre-devel zlib-devel
+```
+* MacOSX: port install glib2 pcre 
+One needs to install MySQL/Percona/MariaDB development versions:
+* Ubuntu or Debian: 
+```
+apt-get install libmysqlclient-dev
+apt-get install libperconaserverclient20-dev
+apt-get install libmariadbclient-dev 
+```
+* Fedora, RedHat and CentOS: 
+```
+yum install -y mysql-devel
+yum install -y Percona-Server-devel-57
+yum install -y mariadb-devel
+```
+CentOS 7 comes by default with MariaDB 5.5 libraries which are very old.
+  It might be better to download a newer version of these libraries (MariaDB, MySQL, Percona etc).
+* openSUSE: 
+```
+zypper install libmysqlclient-devel
+```
+* MacOSX: port install mysql5
+ (You may want to run 'port select mysql mysql5' afterwards)
+
+# How to use mydumper
+
+See [Usage](docs/mydumper_usage.rst)
+
 ## How to install mydumper/myloader?
 
 First get the correct url from the [releases section](https://github.com/maxbube/mydumper/releases) then:
 
 ### RedHat / Centos
 
 ```bash
-yum install https://github.com/maxbube/mydumper/releases/download/v0.9.5/mydumper-0.9.5-1.el7.x86_64.rpm
+yum install https://github.com/maxbube/mydumper/releases/download/v0.10.5/mydumper-0.10.5-1.el7.x86_64.rpm
+yum install https://github.com/maxbube/mydumper/releases/download/v0.10.5/mydumper-0.10.5-1.el8.x86_64.rpm
 ```
 
 ### Ubuntu / Debian
 
 ```bash
-wget https://github.com/maxbube/mydumper/releases/download/v0.9.5/mydumper_0.9.5-1.xenial_amd64.deb
-dpkg -i mydumper_0.9.5-1.xenial_amd64.deb
+wget https://github.com/maxbube/mydumper/releases/download/v0.10.5/mydumper_0.10.5-1.$(lsb_release -cs)_amd64.deb
+dpkg -i mydumper_0.10.5-1.$(lsb_release -cs)_amd64.deb
+```
+
+### OSX
+By using [Homebrew](https://brew.sh/)
+
+```bash
+brew install mydumper
 ```
 
 ## How to build it?
@@ -31,15 +94,6 @@ cmake .
 make
 ```
 
-One needs to install development versions of required libaries (MySQL, GLib, ZLib, PCRE):
-NOTE: you must use the correspondent mysql devel package.
-
-* Ubuntu or Debian: apt-get install libglib2.0-dev libmysqlclient15-dev zlib1g-dev libpcre3-dev libssl-dev
-* Fedora, RedHat and CentOS: yum install glib2-devel mysql-devel zlib-devel pcre-devel openssl-devel
-* openSUSE: zypper install glib2-devel libmysqlclient-devel pcre-devel zlib-devel
-* MacOSX: port install glib2 mysql5 pcre pkgconfig cmake
- (You may want to run 'port select mysql mysql5' afterwards)
-
 One has to make sure, that pkg-config, mysql_config, pcre-config are all in $PATH
 
 Binlog dump is disabled by default to compile with it you need to add -DWITH_BINLOG=ON to cmake options
@@ -51,7 +105,7 @@ To build against mysql libs < 5.7 you need to disable SSL adding -DWITH_SSL=OFF
 This is all done following best MySQL practices and traditions:
 
 * As a precaution, slow running queries on the server either abort the dump, or get killed
-* Global write lock is acquired ("FLUSH TABLES WITH READ LOCK")
+* Global read lock is acquired ("FLUSH TABLES WITH READ LOCK")
 * Various metadata is read ("SHOW SLAVE STATUS","SHOW MASTER STATUS")
 * Other threads connect and establish snapshots ("START TRANSACTION WITH CONSISTENT SNAPSHOT")
 ** On pre-4.1.8 it creates dummy InnoDB table, and reads from it.

binlog.c

@@ -29,6 +29,7 @@
 #include <zlib.h>
 #include "mydumper.h"
 #include "binlog.h"
+#include "set_verbose.h"
 
 #define BINLOG_MAGIC "\xfe\x62\x69\x6e"
 

cmake/modules/FindMySQL.cmake

@@ -84,7 +84,7 @@ find_path(MYSQL_INCLUDE_DIR mysql.h
 )
 
 set(TMP_MYSQL_LIBRARIES "")
-set(CMAKE_FIND_LIBRARY_SUFFIXES .so .a .lib .so.1)
+set(CMAKE_FIND_LIBRARY_SUFFIXES .so .lib .so.1 .dylib .a)
 foreach(MY_LIB ${MYSQL_ADD_LIBRARIES})
     find_library("MYSQL_LIBRARIES_${MY_LIB}" NAMES ${MY_LIB}
         HINTS

common.h

@@ -29,6 +29,8 @@ char *cert = NULL;
 char *ca = NULL;
 char *capath = NULL;
 char *cipher = NULL;
+char *tls_version = NULL;
+gchar *ssl_mode = NULL;
 #endif
 
 gboolean askPassword = FALSE;
@@ -65,6 +67,8 @@ GOptionEntry common_entries[] = {
      "Use a specific defaults file", NULL},
 #ifdef WITH_SSL
     {"ssl", 0, 0, G_OPTION_ARG_NONE, &ssl, "Connect using SSL", NULL},
+    {"ssl-mode", 0, 0, G_OPTION_ARG_STRING, &ssl_mode,
+     "Desired security state of the connection to the server: DISABLED, PREFERRED, REQUIRED, VERIFY_CA, VERIFY_IDENTITY", NULL},
     {"key", 0, 0, G_OPTION_ARG_STRING, &key, "The path name to the key file",
      NULL},
     {"cert", 0, 0, G_OPTION_ARG_STRING, &cert,
@@ -77,6 +81,8 @@ GOptionEntry common_entries[] = {
      NULL},
     {"cipher", 0, 0, G_OPTION_ARG_STRING, &cipher,
      "A list of permissible ciphers to use for SSL encryption", NULL},
+    {"tls-version", 0, 0, G_OPTION_ARG_STRING, &tls_version,
+     "Which protocols the server permits for encrypted connections", NULL},
 #endif
     {NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}};
 

connection.c

@@ -28,7 +28,9 @@ extern char *cert;
 extern char *ca;
 extern char *capath;
 extern char *cipher;
+extern char *tls_version;
 extern gboolean ssl;
+extern gchar *ssl_mode;
 #endif
 extern guint compress_protocol;
 
@@ -45,11 +47,48 @@ void configure_connection(MYSQL *conn, const char *name) {
   unsigned int i;
   if (ssl) {
     i = SSL_MODE_REQUIRED;
+    mysql_options(conn, MYSQL_OPT_SSL_MODE, &i);
   } else {
-    i = SSL_MODE_DISABLED;
+    if (ssl_mode) {
+      if (g_ascii_strncasecmp(ssl_mode, "DISABLED", 16) == 0) {
+        i = SSL_MODE_DISABLED;
+      }
+      else if (g_ascii_strncasecmp(ssl_mode, "PREFERRED", 16) == 0) {
+        i = SSL_MODE_PREFERRED;
+      }
+      else if (g_ascii_strncasecmp(ssl_mode, "REQUIRED", 16) == 0) {
+        i = SSL_MODE_REQUIRED;
+      }
+      else if (g_ascii_strncasecmp(ssl_mode, "VERIFY_CA", 16) == 0) {
+        i = SSL_MODE_VERIFY_CA;
+      }
+      else if (g_ascii_strncasecmp(ssl_mode, "VERIFY_IDENTITY", 16) == 0) {
+        i = SSL_MODE_VERIFY_IDENTITY;
+      }
+      else {
+        g_critical("Unsupported ssl-mode specified: %s\n", ssl_mode);
+        exit(EXIT_FAILURE);
+      }
+      mysql_options(conn, MYSQL_OPT_SSL_MODE, &i);
+    }
+  }
+  if (key) {
+    mysql_options(conn, MYSQL_OPT_SSL_KEY, key);
+  }
+  if (cert) {
+    mysql_options(conn, MYSQL_OPT_SSL_CERT, cert);
+  }
+  if (ca) {
+    mysql_options(conn, MYSQL_OPT_SSL_CA, ca);
+  }
+  if (capath) {
+    mysql_options(conn, MYSQL_OPT_SSL_CAPATH, capath);
+  }
+  if (cipher) {
+    mysql_options(conn, MYSQL_OPT_SSL_CIPHER, cipher);
+  }
+  if (tls_version) {
+    mysql_options(conn, MYSQL_OPT_TLS_VERSION, tls_version);
   }
-
-  mysql_ssl_set(conn, key, cert, ca, capath, cipher);
-  mysql_options(conn, MYSQL_OPT_SSL_MODE, &i);
 #endif
 }

docs/_build/conf.py.in

@@ -120,7 +120,7 @@ html_theme = 'default'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_static']
+html_static_path = ['@CMAKE_CURRENT_SOURCE_DIR@/html/_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

docs/files.rst

@@ -36,7 +36,7 @@ Where 'chunk' is a number padded with up to 5 zeros.
 
 Table Schemas
 -------------
-When the :option:`--schemas <mydumper --schemas>` option is used mydumper will
+As long as the :option:`--no-schemas <mydumper --no-schemas>` option is not specified, mydumper will
 create a file for the schema of every table it is writing data for.  The files
 for this are in the following format::
 

docs/mydumper_usage.rst

@@ -35,6 +35,13 @@ The :program:`mydumper` tool has several available options:
 .. option:: --defaults-file
 
    Use the given option file. If the file does not exist or is otherwise inaccessible, no failure occurs
+   
+   Its possible to authenticate via --defaults-file.
+   A section [mydumper]/[myloader] or [client] is needed for authentication
+   
+   [mydumper]
+   user=
+   password=
 
 .. option:: --host, -h
 
@@ -114,6 +121,10 @@ The :program:`mydumper` tool has several available options:
 
    Comma separated list of storage engines to ignore
 
+.. option:: --insert-ignore, -N
+
+   Dump rows with INSERT IGNORE INTO instead of INSERT INTO
+
 .. option:: --no-schemas, -m
 
    Do not dump schemas with the data
@@ -138,6 +149,14 @@ The :program:`mydumper` tool has several available options:
 
    Do not dump views
 
+.. option:: --long-query-retries
+
+   Retry checking for long queries, default 0 (do not retry)
+
+.. option:: --long-query-retry-interval
+
+   Time to wait before retrying the long query check in seconds, default 60
+
 .. option:: --long-query-guard, -l
 
    Timeout for long query execution in seconds, default 60

logging.c

@@ -0,0 +1,77 @@
+/*
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+        Authors:        Domas Mituzas, Facebook ( domas at fb dot com )
+                        Mark Leith, Oracle Corporation (mark dot leith at oracle
+   dot com) Andrew Hutchings, SkySQL (andrew at skysql dot com)
+
+*/
+
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <glib.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <time.h>
+#include <zlib.h>
+#include <pcre.h>
+#include <signal.h>
+#include <glib/gstdio.h>
+
+gchar *logfile;
+FILE *logoutfile;
+
+void no_log(const gchar *log_domain, GLogLevelFlags log_level,
+            const gchar *message, gpointer user_data) {
+  (void)log_domain;
+  (void)log_level;
+  (void)message;
+  (void)user_data;
+}
+
+void write_log_file(const gchar *log_domain, GLogLevelFlags log_level,
+                    const gchar *message, gpointer user_data) {
+  (void)log_domain;
+  (void)user_data;
+
+  gchar date[20];
+  time_t rawtime;
+  struct tm timeinfo;
+
+  time(&rawtime);
+  localtime_r(&rawtime, &timeinfo);
+  strftime(date, 20, "%Y-%m-%d %H:%M:%S", &timeinfo);
+
+  GString *message_out = g_string_new(date);
+  if (log_level & G_LOG_LEVEL_DEBUG) {
+    g_string_append(message_out, " [DEBUG] - ");
+  } else if ((log_level & G_LOG_LEVEL_INFO) ||
+             (log_level & G_LOG_LEVEL_MESSAGE)) {
+    g_string_append(message_out, " [INFO] - ");
+  } else if (log_level & G_LOG_LEVEL_WARNING) {
+    g_string_append(message_out, " [WARNING] - ");
+  } else if ((log_level & G_LOG_LEVEL_ERROR) ||
+             (log_level & G_LOG_LEVEL_CRITICAL)) {
+    g_string_append(message_out, " [ERROR] - ");
+  }
+
+  g_string_append_printf(message_out, "%s\n", message);
+  if (write(fileno(logoutfile), message_out->str, message_out->len) <= 0) {
+    fprintf(stderr, "Cannot write to log file with error %d.  Exiting...",
+            errno);
+  }
+  g_string_free(message_out, TRUE);
+}