Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor order of actions in guest reboot implementations #3469

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 19 additions & 18 deletions tmt/steps/execute/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -484,16 +484,18 @@ def handle_reboot(self) -> bool:
f" with reboot count {self._reboot_count}"
f" and test restart count {self._restart_count}.")

reboot_command: Optional[ShellScript] = None
timeout: Optional[int] = None
rebooted = False

if self.hard_reboot_requested:
pass
rebooted = self.guest.reboot(hard=True)

elif self.soft_reboot_requested:
# Extract custom hints from the file, and reset it.
reboot_data = json.loads(self.reboot_request_path.read_text())

reboot_command: Optional[ShellScript] = None
timeout: Optional[int] = None

if reboot_data.get('command'):
with suppress(TypeError):
reboot_command = ShellScript(reboot_data.get('command'))
Expand All @@ -506,25 +508,24 @@ def handle_reboot(self) -> bool:
os.remove(self.reboot_request_path)
self.guest.push(self.test_data_path)

rebooted = False

try:
rebooted = self.guest.reboot(
hard=self.hard_reboot_requested,
command=reboot_command,
timeout=timeout)
try:
rebooted = self.guest.reboot(
hard=False,
command=reboot_command,
timeout=timeout)

except tmt.utils.RunError:
self.logger.fail(
f"Failed to reboot guest using the custom command '{reboot_command}'.")
except tmt.utils.RunError:
if reboot_command is not None:
self.logger.fail(
f"Failed to reboot guest using the custom command '{reboot_command}'.")

raise
raise

except tmt.utils.ProvisionError:
self.logger.warning(
"Guest does not support soft reboot, trying hard reboot.")
except tmt.steps.provision.RebootModeNotSupportedError:
self.logger.warning(
"Guest does not support soft reboot, trying hard reboot.")

rebooted = self.guest.reboot(hard=True, timeout=timeout)
rebooted = self.guest.reboot(hard=True, timeout=timeout)

if not rebooted:
raise tmt.utils.RebootTimeoutError("Reboot timed out.")
Expand Down
132 changes: 111 additions & 21 deletions tmt/steps/provision/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,29 @@ def format_guest_full_name(name: str, role: Optional[str]) -> str:
return f'{name} ({role})'


class RebootModeNotSupportedError(ProvisionError):
""" A requested reboot mode is not supported by the guest """

def __init__(
self,
message: Optional[str] = None,
guest: Optional['Guest'] = None,
hard: bool = False,
*args: Any,
**kwargs: Any) -> None:

if message is not None:
pass

elif guest is not None:
message = f"Guest '{guest.multihost_name}' does not support {'hard' if hard else 'soft'} reboot." # noqa: E501

else:
message = f"Guest does not support {'hard' if hard else 'soft'} reboot."

super().__init__(message, *args, **kwargs)


class CheckRsyncOutcome(enum.Enum):
ALREADY_INSTALLED = 'already-installed'
INSTALLED = 'installed'
Expand Down Expand Up @@ -1389,23 +1412,54 @@ def stop(self) -> None:

raise NotImplementedError

@overload
def reboot(
self,
hard: Literal[True] = True,
command: None = None,
timeout: Optional[int] = None,
tick: float = tmt.utils.DEFAULT_WAIT_TICK,
tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> bool:
pass

@overload
def reboot(
self,
hard: Literal[False] = False,
command: Optional[Union[Command, ShellScript]] = None,
timeout: Optional[int] = None,
tick: float = tmt.utils.DEFAULT_WAIT_TICK,
tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> bool:
pass

def reboot(
self,
hard: bool = False,
command: Optional[Union[Command, ShellScript]] = None,
timeout: Optional[int] = None) -> bool:
timeout: Optional[int] = None,
tick: float = tmt.utils.DEFAULT_WAIT_TICK,
tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> bool:
"""
Reboot the guest, return True if successful
Reboot the guest, and wait for the guest to recover.

Parameter 'hard' set to True means that guest should be
rebooted by way which is not clean in sense that data can be
lost. When set to False reboot should be done gracefully.
.. note::

Use the 'command' parameter to specify a custom reboot command
instead of the default 'reboot'.
Custom reboot command can be used only in combination with a
soft reboot. If both ``hard`` and ``command`` are set, a hard
reboot will be requested, and ``command`` will be ignored.

Parameter 'timeout' can be used to specify time (in seconds) to
wait for the guest to come back up after rebooting.
:param hard: if set, force the reboot. This may result in a loss
of data. The default of ``False`` will attempt a graceful
reboot.
:param command: a command to run on the guest to trigger the
reboot. If ``hard`` is also set, ``command`` is ignored.
:param timeout: amount of time in which the guest must become available
again.
:param tick: how many seconds to wait between two consecutive attempts
of contacting the guest.
:param tick_increase: a multiplier applied to ``tick`` after every
attempt.
:returns: ``True`` if the reboot succeeded, ``False`` otherwise.
"""

raise NotImplementedError
Expand Down Expand Up @@ -2126,22 +2180,31 @@ def stop(self) -> None:
self._unlink_ssh_master_socket_path()

def perform_reboot(self,
command: Callable[[], tmt.utils.CommandOutput],
action: Callable[[], Any],
timeout: Optional[int] = None,
tick: float = tmt.utils.DEFAULT_WAIT_TICK,
tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE,
hard: bool = False) -> bool:
fetch_boot_time: bool = True) -> bool:
"""
Perform the actual reboot and wait for the guest to recover.

:param command: a callable running the actual command triggering
the reboot.
This is the core implementation of the common task of triggering
a reboot and waiting for the guest to recover. :py:meth:`reboot`
is the public API of guest classes, and feeds
:py:meth:`perform_reboot` with the right ``action`` callable.

:param action: a callable which will trigger the requested reboot.
:param timeout: amount of time in which the guest must become available
again.
:param tick: how many seconds to wait between two consecutive attempts
of contacting the guest.
:param tick_increase: a multiplier applied to ``tick`` after every
attempt.
:param fetch_boot_time: if set, the current boot time of the
guest would be read first, and used for testing whether the
reboot has been performed. This will require communication
with the guest, therefore it is recommended to use ``False``
with hard reboot of unhealthy guests.
:returns: ``True`` if the reboot succeeded, ``False`` otherwise.
"""

Expand All @@ -2157,10 +2220,10 @@ def get_boot_time() -> int:

return int(match.group(1))

current_boot_time = 0 if hard else get_boot_time()
current_boot_time = get_boot_time() if fetch_boot_time else 0

try:
command()
action()

except tmt.utils.RunError as error:
# Connection can be closed by the remote host even before the
Expand Down Expand Up @@ -2205,6 +2268,26 @@ def check_boot_time() -> None:
self.debug("Connection to guest succeeded after reboot.")
return True

@overload
def reboot(
self,
hard: Literal[True] = True,
command: None = None,
timeout: Optional[int] = None,
tick: float = tmt.utils.DEFAULT_WAIT_TICK,
tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> bool:
pass

@overload
def reboot(
self,
hard: Literal[False] = False,
command: Optional[Union[Command, ShellScript]] = None,
timeout: Optional[int] = None,
tick: float = tmt.utils.DEFAULT_WAIT_TICK,
tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> bool:
pass

def reboot(
self,
hard: bool = False,
Expand All @@ -2215,9 +2298,17 @@ def reboot(
"""
Reboot the guest, and wait for the guest to recover.

:param hard: if set, force the reboot. This may result in a loss of
data. The default of ``False`` will attempt a graceful reboot.
:param command: a command to run on the guest to trigger the reboot.
.. note::

Custom reboot command can be used only in combination with a
soft reboot. If both ``hard`` and ``command`` are set, a hard
reboot will be requested, and ``command`` will be ignored.

:param hard: if set, force the reboot. This may result in a loss
of data. The default of ``False`` will attempt a graceful
reboot.
:param command: a command to run on the guest to trigger the
reboot. If ``hard`` is also set, ``command`` is ignored.
:param timeout: amount of time in which the guest must become available
again.
:param tick: how many seconds to wait between two consecutive attempts
Expand All @@ -2233,14 +2324,13 @@ def reboot(

actual_command = command or DEFAULT_REBOOT_COMMAND

self.debug(f"Reboot using the command '{actual_command}'.")
self.debug(f"Soft reboot using command '{actual_command}'.")

return self.perform_reboot(
lambda: self.execute(actual_command),
timeout=timeout,
tick=tick,
tick_increase=tick_increase,
hard=hard)
tick_increase=tick_increase)

def remove(self) -> None:
"""
Expand Down
4 changes: 4 additions & 0 deletions tmt/steps/provision/bootc.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ class ProvisionBootc(tmt.steps.provision.ProvisionPlugin[BootcData]):
The bootc disk creation requires running podman as root. The plugin will
automatically check if the current podman connection is rootless. If it is,
a podman machine will be spun up and used to build the bootc disk.

To trigger hard reboot of a guest, plugin uses testcloud API. It is
also used to trigger soft reboot unless a custom reboot command was
specified via ``tmt-reboot -c ...``.
"""

_data_class = BootcData
Expand Down
Loading
Loading