From 705c6c2569b88970bcdc2af08c2f3ca25c87c39b Mon Sep 17 00:00:00 2001 From: Timo Pollmeier Date: Tue, 26 Nov 2024 10:12:32 +0100 Subject: [PATCH] Fix: Avoid DB check inserts in cleanup-sequences When running gvmd with the --optimize=cleanup-sequences option, the database checks during initialization will avoid most inserts. Setting database_version and max_hosts in the meta table now uses INSERT ... ON CONFLICT instead of deleting the previous entry. Skipping the inserts prevents the cleanup failing if one of the sequence has already run out of ids. The change to the meta table update reduces the risk of the meta table id sequence running out. --- src/manage.c | 9 +- src/manage.h | 2 +- src/manage_sql.c | 142 ++++++++++++++++++++++---------- src/manage_sql.h | 2 +- src/manage_sql_configs.c | 7 +- src/manage_sql_configs.h | 2 +- src/manage_sql_nvts.c | 6 +- src/manage_sql_port_lists.c | 7 +- src/manage_sql_report_formats.c | 11 ++- src/manage_sql_report_formats.h | 2 +- src/manage_sql_secinfo.c | 3 +- 11 files changed, 135 insertions(+), 58 deletions(-) diff --git a/src/manage.c b/src/manage.c index c194bae68..f9e0899c4 100644 --- a/src/manage.c +++ b/src/manage.c @@ -978,7 +978,8 @@ int manage_create_encryption_key (GSList *log_config, const db_conn_info_t *database) { - int ret = manage_option_setup (log_config, database); + int ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) { printf ("Error setting up log config or database connection."); @@ -1042,7 +1043,8 @@ manage_set_encryption_key (GSList *log_config, const db_conn_info_t *database, const char *uid) { - int ret = manage_option_setup (log_config, database); + int ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) { printf ("Error setting up log config or database connection.\n"); @@ -5707,7 +5709,8 @@ manage_rebuild_gvmd_data_from_feed (const char *types, return -1; } - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) { if (error_msg) diff --git a/src/manage.h b/src/manage.h index 0285d708d..8882d7a52 100644 --- a/src/manage.h +++ b/src/manage.h @@ -127,7 +127,7 @@ init_manage (GSList*, const db_conn_info_t *, int, int, int, int, manage_connection_forker_t, int); int -init_manage_helper (GSList *, const db_conn_info_t *, int); +init_manage_helper (GSList *, const db_conn_info_t *, int, int); void init_manage_process (const db_conn_info_t*); diff --git a/src/manage_sql.c b/src/manage_sql.c index 945ae9aa0..6e653b13d 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -930,13 +930,15 @@ cert_check_time () * * @param[in] log_config Log configuration. * @param[in] database Database. + * @param[in] avoid_db_check_inserts Whether to avoid inserts in DB check. * * @return 0 success, -1 error, -2 database is too old, * -3 database needs to be initialised from server, * -5 database is too new. */ int -manage_option_setup (GSList *log_config, const db_conn_info_t *database) +manage_option_setup (GSList *log_config, const db_conn_info_t *database, + int avoid_db_check_inserts) { int ret; @@ -947,7 +949,8 @@ manage_option_setup (GSList *log_config, const db_conn_info_t *database) } ret = init_manage_helper (log_config, database, - MANAGE_ABSOLUTE_MAX_IPS_PER_TARGET); + MANAGE_ABSOLUTE_MAX_IPS_PER_TARGET, + avoid_db_check_inserts); assert (ret != -4); switch (ret) { @@ -6159,10 +6162,9 @@ manage_cert_db_version () void set_db_version (int version) { - sql ("DELETE FROM %s.meta WHERE name = 'database_version';", - sql_schema ()); sql ("INSERT INTO %s.meta (name, value)" - " VALUES ('database_version', '%i');", + " VALUES ('database_version', '%i')" + " ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value;", sql_schema (), version); } @@ -6408,7 +6410,8 @@ manage_encrypt_all_credentials (GSList *log_config, g_info (" (Re-)encrypting all credentials."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -6449,7 +6452,8 @@ manage_decrypt_all_credentials (GSList *log_config, g_info (" Decrypting all credentials."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -6712,7 +6716,8 @@ manage_check_alerts (GSList *log_config, const db_conn_info_t *database) g_info (" Checking alerts."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -15835,11 +15840,16 @@ manage_update_nvti_cache () /** * @brief Ensure the predefined scanner exists. * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. + * * @return 0 if success, -1 if error. */ static int -check_db_scanners () +check_db_scanners (int avoid_db_check_inserts) { + if (avoid_db_check_inserts) + return 0; + if (sql_int ("SELECT count(*) FROM scanners WHERE uuid = '%s';", SCANNER_UUID_DEFAULT) == 0) { @@ -15868,11 +15878,16 @@ check_db_scanners () /** * @brief Initialize the default settings. * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. + * * Ensure all the default manager settings exist. */ static void -check_db_settings () +check_db_settings (int avoid_db_check_inserts) { + if (avoid_db_check_inserts) + return; + if (sql_int ("SELECT count(*) FROM settings" " WHERE uuid = '6765549a-934e-11e3-b358-406186ea4fc5'" " AND " ACL_IS_GLOBAL () ";") @@ -16219,10 +16234,15 @@ check_db_versions () /** * @brief Ensures the sanity of nvts cache in DB. + * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. */ static void -check_db_nvt_selectors () +check_db_nvt_selectors (int avoid_db_check_inserts) { + if (avoid_db_check_inserts) + return; + /* Ensure every part of the predefined selector exists. * This restores entries lost due to the error solved 2010-08-13 by r8805. */ if (sql_int ("SELECT count(*) FROM nvt_selectors WHERE name =" @@ -16359,10 +16379,15 @@ add_permissions_on_globals (const gchar *role_uuid) /** * @brief Ensure the predefined permissions exists. + * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. */ static void -check_db_permissions () +check_db_permissions (int avoid_db_check_inserts) { + if (avoid_db_check_inserts) + return; + command_t *command; if (sql_int ("SELECT count(*) FROM permissions" @@ -16521,10 +16546,15 @@ check_db_permissions () /** * @brief Ensure the predefined roles exists. + * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. */ static void -check_db_roles () +check_db_roles (int avoid_db_check_inserts) { + if (avoid_db_check_inserts) + return; + if (sql_int ("SELECT count(*) FROM roles WHERE uuid = '" ROLE_UUID_ADMIN "';") == 0) sql ("INSERT INTO roles" @@ -16680,11 +16710,11 @@ manage_migrate_relay_sensors () * Only called by init_manage_internal, and ultimately only by the main process. * * @param[in] check_encryption_key Whether to check encryption key. - * + * @param[in] avoid_db_check_inserts Whether to avoid inserts in DB check. * @return 0 success, -1 error. */ static int -check_db (int check_encryption_key) +check_db (int check_encryption_key, int avoid_db_check_inserts) { /* The file locks managed at startup ensure that this is the only Manager * process accessing the db. Nothing else should be accessing the db, access @@ -16695,19 +16725,19 @@ check_db (int check_encryption_key) create_tables (); check_db_sequences (); set_db_version (GVMD_DATABASE_VERSION); - check_db_roles (); - check_db_nvt_selectors (); + check_db_roles (avoid_db_check_inserts); + check_db_nvt_selectors (avoid_db_check_inserts); check_db_nvts (); - check_db_port_lists (); + check_db_port_lists (avoid_db_check_inserts); clean_auth_cache (); - if (check_db_scanners ()) + if (check_db_scanners (avoid_db_check_inserts)) goto fail; - if (check_db_report_formats ()) + if (check_db_report_formats (avoid_db_check_inserts)) goto fail; if (check_db_report_formats_trash ()) goto fail; - check_db_permissions (); - check_db_settings (); + check_db_permissions (avoid_db_check_inserts); + check_db_settings (avoid_db_check_inserts); cleanup_schedule_times (); if (check_encryption_key && check_db_encryption_key ()) goto fail; @@ -16874,6 +16904,7 @@ cleanup_tables () * with GMP when an alert occurs. * @param[in] skip_db_check Skip DB check. * @param[in] check_encryption_key Check encryption key if doing DB check. + * @param[in] avoid_db_check_inserts Whether to avoid inserts in DB check. * * @return 0 success, -1 error, -2 database is too old, * -4 max_ips_per_target out of range, -5 database is too new. @@ -16888,7 +16919,8 @@ init_manage_internal (GSList *log_config, int stop_tasks, manage_connection_forker_t fork_connection, int skip_db_check, - int check_encryption_key) + int check_encryption_key, + int avoid_db_check_inserts) { int ret; @@ -16974,7 +17006,7 @@ init_manage_internal (GSList *log_config, * 2 a helper processes (--create-user, --get-users, etc) when the * main process is not running. */ - ret = check_db (check_encryption_key); + ret = check_db (check_encryption_key, avoid_db_check_inserts); if (ret) return ret; @@ -16982,8 +17014,10 @@ init_manage_internal (GSList *log_config, /* Set max_hosts in db, so database server side can access it. */ - sql ("DELETE FROM meta WHERE name = 'max_hosts';"); - sql ("INSERT INTO meta (name, value) VALUES ('max_hosts', %i);", max_hosts); + sql ("INSERT INTO meta (name, value)" + " VALUES ('max_hosts', %i)" + " ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value;", + max_hosts); } if (stop_tasks) @@ -16997,7 +17031,7 @@ init_manage_internal (GSList *log_config, if (skip_db_check == 0) /* Requires NVT cache. */ - check_db_configs (); + check_db_configs (avoid_db_check_inserts); sql_close (); gvmd_db_conn_info.name = database->name ? g_strdup (database->name) : NULL; @@ -17051,7 +17085,8 @@ init_manage (GSList *log_config, const db_conn_info_t *database, 1, /* Stop active tasks. */ fork_connection, skip_db_check, - 1); /* Check encryption key if checking db. */ + 1, /* Check encryption key if checking db. */ + 0 /* Do not avoid inserts if checking db. */); } /** @@ -17063,7 +17098,8 @@ init_manage (GSList *log_config, const db_conn_info_t *database, * * @param[in] log_config Log configuration. * @param[in] database Location of database. - * @param[in] max_ips_per_target Max number of IPs per target. + * @param[in] max_ips_per_target Max number of IPs per target. + * @param[in] avoid_db_check_inserts Whether to avoid inserts in DB check. * * @return 0 success, -1 error, -2 database is too old, -3 database needs * to be initialised from server, -4 max_ips_per_target out of range, @@ -17071,7 +17107,7 @@ init_manage (GSList *log_config, const db_conn_info_t *database, */ int init_manage_helper (GSList *log_config, const db_conn_info_t *database, - int max_ips_per_target) + int max_ips_per_target, int avoid_db_check_inserts) { return init_manage_internal (log_config, database, @@ -17088,7 +17124,8 @@ init_manage_helper (GSList *log_config, const db_conn_info_t *database, lockfile_locked ("gvm-serving") ? 1 /* Skip DB check. */ : 0, /* Do DB check. */ - 0); /* Dummy. */ + 0, /* Dummy. */ + avoid_db_check_inserts); } /** @@ -40245,7 +40282,8 @@ manage_create_scanner (GSList *log_config, const db_conn_info_t *database, g_info (" Creating scanner."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -40434,7 +40472,8 @@ manage_delete_scanner (GSList *log_config, const db_conn_info_t *database, return 3; } - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -40502,7 +40541,8 @@ manage_modify_scanner (GSList *log_config, const db_conn_info_t *database, g_info (" Modifying scanner."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -40721,7 +40761,8 @@ manage_verify_scanner (GSList *log_config, const db_conn_info_t *database, g_info (" Verifying scanner."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -42206,7 +42247,8 @@ manage_get_scanners (GSList *log_config, const db_conn_info_t *database) g_info (" Getting scanners."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -46145,7 +46187,8 @@ manage_get_roles (GSList *log_config, const db_conn_info_t *database, g_info (" Getting roles."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -52790,7 +52833,8 @@ manage_modify_setting (GSList *log_config, const db_conn_info_t *database, return 3; } - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -52944,7 +52988,8 @@ manage_create_user (GSList *log_config, const db_conn_info_t *database, g_info (" Creating user."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -53034,7 +53079,8 @@ manage_delete_user (GSList *log_config, const db_conn_info_t *database, g_info (" Deleting user."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -53099,7 +53145,8 @@ manage_get_users (GSList *log_config, const db_conn_info_t *database, g_info (" Getting users."); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -53203,7 +53250,8 @@ manage_set_password (GSList *log_config, const db_conn_info_t *database, return -1; } - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return ret; @@ -58214,7 +58262,15 @@ manage_optimize (GSList *log_config, const db_conn_info_t *database, return 1; } - ret = manage_option_setup (log_config, database); + int avoid_db_check_inserts = 0; + /* The optimize=cleanup-sequences option may be used if a sequence has + * already reached its maximum value, so avoid any inserts that may cause + * a sequence maximum error. * + */ + if (strcasecmp (name, "cleanup-sequences") == 0) + avoid_db_check_inserts = 1; + + ret = manage_option_setup (log_config, database, avoid_db_check_inserts); if (ret) return ret; diff --git a/src/manage_sql.h b/src/manage_sql.h index c98e15adc..d24868e61 100644 --- a/src/manage_sql.h +++ b/src/manage_sql.h @@ -448,7 +448,7 @@ void check_alerts (); int -manage_option_setup (GSList *, const db_conn_info_t *); +manage_option_setup (GSList *, const db_conn_info_t *, int); void manage_option_cleanup (); diff --git a/src/manage_sql_configs.c b/src/manage_sql_configs.c index 28e2bb10e..07e25734e 100644 --- a/src/manage_sql_configs.c +++ b/src/manage_sql_configs.c @@ -4532,12 +4532,17 @@ update_config (config_t config, const gchar *name, /** * @brief Check configs, for startup. + * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. */ void -check_db_configs () +check_db_configs (int avoid_db_check_inserts) { migrate_predefined_configs (); + if (avoid_db_check_inserts) + return; + if (sync_configs_with_feed (FALSE) <= -1) g_warning ("%s: Failed to sync configs with feed", __func__); diff --git a/src/manage_sql_configs.h b/src/manage_sql_configs.h index 888d4e4fb..901878423 100644 --- a/src/manage_sql_configs.h +++ b/src/manage_sql_configs.h @@ -97,7 +97,7 @@ update_config (config_t, const gchar *, const gchar *, const gchar *, int, const array_t*, const array_t*, const gchar *); void -check_db_configs (); +check_db_configs (int); void check_whole_only_in_configs (); diff --git a/src/manage_sql_nvts.c b/src/manage_sql_nvts.c index 04e9cf3c8..73aab286c 100644 --- a/src/manage_sql_nvts.c +++ b/src/manage_sql_nvts.c @@ -2654,7 +2654,8 @@ manage_rebuild (GSList *log_config, const db_conn_info_t *database) return -1; } - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) { feed_lockfile_unlock (&lockfile); @@ -2727,7 +2728,8 @@ manage_dump_vt_verification (GSList *log_config, return -1; } - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) { feed_lockfile_unlock (&lockfile); diff --git a/src/manage_sql_port_lists.c b/src/manage_sql_port_lists.c index c24abaea6..b56b9fcc2 100644 --- a/src/manage_sql_port_lists.c +++ b/src/manage_sql_port_lists.c @@ -2652,12 +2652,17 @@ update_port_list (port_list_t port_list, const gchar *name, /** * @brief Check port lists, for startup. + * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. */ void -check_db_port_lists () +check_db_port_lists (int avoid_db_check_inserts) { migrate_predefined_port_lists (); + if (avoid_db_check_inserts) + return; + if (sync_port_lists_with_feed (FALSE) <= -1) g_warning ("%s: Failed to sync port lists with feed", __func__); diff --git a/src/manage_sql_report_formats.c b/src/manage_sql_report_formats.c index becfcb4c5..0a1b49b39 100644 --- a/src/manage_sql_report_formats.c +++ b/src/manage_sql_report_formats.c @@ -4895,16 +4895,21 @@ check_db_trash_report_formats () /** * @brief Ensure the predefined report formats exist. * + * @param[in] avoid_db_check_inserts Whether to avoid inserts. + * * @return 0 success, -1 error. */ int -check_db_report_formats () +check_db_report_formats (int avoid_db_check_inserts) { if (migrate_predefined_report_formats ()) return -1; - if (sync_report_formats_with_feed (FALSE) <= -1) - g_warning ("%s: Failed to sync report formats with feed", __func__); + if (avoid_db_check_inserts == 0) + { + if (sync_report_formats_with_feed (FALSE) <= -1) + g_warning ("%s: Failed to sync report formats with feed", __func__); + } if (check_db_trash_report_formats ()) return -1; diff --git a/src/manage_sql_report_formats.h b/src/manage_sql_report_formats.h index 1ea556752..de7a04be2 100644 --- a/src/manage_sql_report_formats.h +++ b/src/manage_sql_report_formats.h @@ -83,7 +83,7 @@ int migrate_predefined_report_formats (); int -check_db_report_formats (); +check_db_report_formats (int); int check_db_report_formats_trash (); diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 48cf21772..c80247780 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -5754,7 +5754,8 @@ manage_rebuild_scap (GSList *log_config, const db_conn_info_t *database) g_info (" Rebuilding SCAP data"); - ret = manage_option_setup (log_config, database); + ret = manage_option_setup (log_config, database, + 0 /* avoid_db_check_inserts */); if (ret) return -1;