WARNING Server::check_worker_exit_status(): worker(pid= , id=4) abnormal exit, status=0, signal=9 #4776
Comments
|
This means that the process was forcibly killed by OS with SIGKILL ( Did you execute kill -9 yourself, or an OOM kill occurred. Find key info in dmesg log ( |
|
According to the strace log you provided, it may be that the worker process shutdown timed out and was forcibly killed by the main process. Please provide more strace logs before |
|
Hello @matyhtf. No, I ran "kill -9". I can't use dmesg as I'm working on Docker. I am sending you a larger strace log file. I also wrote the details here: https://medium.com/mikrogovernment/strace-ile-uygulama-davran%C4%B1%C5%9F%C4%B1-inceleme-72e69cd84151 |
|
The "kill log" at 2894810 and 2894814 line |
|
I have same issue when running GRPC client. |
|
i have the same issue , |
|
Run into the same issue Worker: 0 OS: Linux 5.4.17-2136.302.6.1.el7uek.x86_64 #2 SMP Tue Dec 14 13:21:01 PST 2021 x86_64 |
|
[INFO] HTTP Server listening at 0.0.0.0:9820
OS: Linux 3.10.0-1160.36.2.el7.x86_64 #1 SMP Wed Jul 21 11:57:15 UTC 2021 x86_64 |
|
@wilbur-yu Can you provide a simple script for reproducing the error? |
生产项目遇到的情况, 目前还没定位到具体触发代码, 项目是基于Hyperf开发的. 遇到这个问题是刚升级至Hyperf 3.0后开始出现的. |
全局使用该中间件后, 稳定复现. <?php
declare(strict_types=1);
/**
* This file is part of project burton.
*
* @author [email protected]
*
* @link https://github.com/wilbur-yu
*/
namespace Common\Middleware;
use Carbon\Carbon;
use Common\Kernel\Log\AppendRequestProcessor;
use Common\Report\Notifier;
use Common\Support\Traits\HasUser;
use Hyperf\Codec\Json;
use Hyperf\Context\Context;
use Hyperf\Coroutine\Coroutine;
use Psr\Container\ContainerInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Server\MiddlewareInterface;
use Psr\Http\Server\RequestHandlerInterface;
use Throwable;
use function Hyperf\Config\config;
use function Hyperf\Coroutine\defer;
use function Hyperf\Support\make;
use function microtime;
use function str_contains;
class DebugMiddleware implements MiddlewareInterface
{
use HasUser;
public function __construct(protected ContainerInterface $container)
{
}
/**
* @param \Psr\Http\Message\ServerRequestInterface $request
* @param \Psr\Http\Server\RequestHandlerInterface $handler
*
* @throws \Psr\Container\ContainerExceptionInterface
* @throws \Psr\Container\NotFoundExceptionInterface
* @return \Psr\Http\Message\ResponseInterface
*/
public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
{
logger()->info($request->getHeaderLine('accept'));
if ($request->getMethod() === 'HEAD' && $request->getUri()->getPath() === '/api') {
return $handler->handle($request);
}
Context::getOrSet(AppendRequestProcessor::LOG_REQUEST_ID_KEY, uniqid('', true));
Context::getOrSet(AppendRequestProcessor::LOG_COROUTINE_ID_KEY, Coroutine::id());
try {
$this->collecter($request);
} finally {
defer(function (): void {
try {
$this->record();
} catch (Throwable $e) {
logger('debug.middleware')->error($e->getMessage().' '.format_throwable($e));
}
});
}
return $handler->handle($request);
}
/**
* @param \Psr\Http\Message\ServerRequestInterface|null $request
*
* @throws \Psr\Container\ContainerExceptionInterface
* @throws \Psr\Container\NotFoundExceptionInterface
* @return void
*/
protected function collecter(?ServerRequestInterface $request): void
{
$startTime = microtime(true);
$context = [
'app_name' => config('app_name'),
'app_env' => config('app_env'),
'trigger_time' => $startTime,
'usage_memory' => round(memory_get_peak_usage(true) / 1024 / 1024, 1).'M',
'url' => $request?->url(),
'uri' => $request?->getUri()->getPath(),
'method' => $request?->getMethod(),
'client_ip' => get_client_ip(),
'duration' => '',
'headers' => $this->getHeaders($request),
'query' => $request?->getQueryParams(),
'payload' => $request?->getParsedBody(),
'user' => $this->getUser(),
];
Context::set(AppendRequestProcessor::LOG_LIFECYCLE_KEY, $context);
}
protected function getHeaders(?ServerRequestInterface $request): array
{
if ($request === null) {
return [];
}
$onlyHeaderKeys = [
'content-type',
'user-agent',
'sign',
'token',
'x-token',
'authorization',
'x-real-ip',
'x-forwarded-for',
'cookie',
];
$logHeaders = [];
foreach ($onlyHeaderKeys as $value) {
if ($request->getHeaderLine($value)) {
$logHeaders[$value] = $request->getHeaderLine($value);
}
}
return array_filter($logHeaders);
}
protected function getUser(): array
{
if (self::isLogin()) {
$user = self::user();
return [
'user' => [
'id' => $user->id,
'nickname' => $user->nickname,
'role' => $user->role,
'ski_resort_id' => $user->ski_resort_id,
'tel' => $user->tel,
],
];
}
return [];
}
public function record(): void
{
$endTime = microtime(true);
$context = Context::get(AppendRequestProcessor::LOG_LIFECYCLE_KEY) ?? [];
$duration = 0;
if (isset($context['trigger_time'])) {
$duration = round(($endTime - $context['trigger_time']) * 1000);
$context['duration'] = $duration.'ms';
$context['trigger_time'] = Carbon::createFromTimestamp($context['trigger_time'])->toDateTimeString();
}
$response = Context::get(ResponseInterface::class);
$context['response'] = $this->getResponseToArray($response);
isset($context['exception']) && make(Notifier::class)->reportForException($context, $context['exception']);
switch (true) {
case isset($context['exception']):
$logLevel = 'error';
break;
case environment()->is('test', 'local'):
$logLevel = 'info';
break;
case $duration >= 500:
$logLevel = 'warning';
break;
default:
$logLevel = 'debug';
}
logger('request')->{$logLevel}(Context::get(AppendRequestProcessor::LOG_REQUEST_ID_KEY), $context);
}
protected function getResponseToArray(?ResponseInterface $response): array
{
if ($response === null) {
return [];
}
$type = $response->getHeaderLine('content-type');
if (str_contains($type, 'application/json')) {
$data = Json::decode($response->getBody()->getContents());
} else {
$data = (array) $response->getBody();
}
if (isset($data['debug']) || isset($data['soar'])) {
unset($data['debug'], $data['soar']);
}
return $data;
}
} |
|
@seddighi78 I faced with the same, looks like |
|
This issue is not fixed yet? I'm having same error when I read a GCP Pub/Sub sub using Swoole and gRPC extension :( |
|
请问你们怎么解决的呢 |
Hello everyone. I have a problem that i thing this is exception. Actualy some issue writed about "abnormal exit" but anyone not continue. Thereforeof i am writing this issue now. I have an application running on Php 8.1 + Laravel 9.2 + Swoole 4.8. Some requests finished with HTTP 408 sometimes. Then when I looked at the log file, I saw the following error.
[2022-07-29 08:59:52 $28.0] WARNING Server::check_worker_exit_status(): worker(pid=49, id=4) abnormal exit, status=0, signal=9then I watched all the processes with "strace" command. I saw the worker killed by manager process. Logs are bellow:
`
ps -aux
www 54 0.0 0.9 473088 39424 ? Sl 12:40 0:00 swoole_http_server: master process
www 65 0.0 0.5 401440 21036 ? S 12:40 0:00 swoole_http_server: manager process
www 1608 0.2 1.3 427668 52384 ? S 13:19 0:00 swoole_http_server: worker process
strace.log:
54 13:27:40.909415 kill(1608, 0) = 0
54 13:27:40.909598 kill(1608, SIGKILL) = 0
1608 13:27:40.943992 +++ killed by SIGKILL +++
65 13:27:40.944275 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_KILLED, si_pid=1608, si_uid=1111, si_status=SIGKILL, si_utime=3840, si_stime=1000} ---
65 13:27:40.944400 rt_sigreturn({mask=[]}) = 1608`
I tryed a lot of things but not fixed this problem. Can you check this issue?
php -v:
PHP 8.1.8 (cli) (built: Jul 11 2022 08:30:39) (NTS)
Copyright (c) The PHP Group
Zend Engine v4.1.8, Copyright (c) Zend Technologies
with Zend OPcache v8.1.8, Copyright (c), by Zend Technologies
php --ri swoole
swoole
Swoole => enabled
Author => Swoole Team [email protected]
Version => 4.8.10
Built => Jul 6 2022 13:14:29
coroutine => enabled with boost asm context
epoll => enabled
eventfd => enabled
signalfd => enabled
cpu_affinity => enabled
spinlock => enabled
rwlock => enabled
sockets => enabled
openssl => OpenSSL 3.0.2 15 Mar 2022
dtls => enabled
http2 => enabled
json => enabled
curl-native => enabled
pcre => enabled
mutex_timedlock => enabled
pthread_barrier => enabled
futex => enabled
mysqlnd => enabled
async_redis => enabled
Directive => Local Value => Master Value
swoole.enable_coroutine => On => On
swoole.enable_library => On => On
swoole.enable_preemptive_scheduler => Off => Off
swoole.display_errors => On => On
swoole.use_shortname => On => On
swoole.unixsock_buffer_size => 8388608 => 8388608
php artisan --version
Laravel Framework 9.21.3
uname -a
Linux 9968d896fe6c 5.15.0-41-generic #44-Ubuntu SMP Wed Jun 22 14:20:53 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
gcc -v
Using built-in specs.
COLLECT_GCC=gcc
COLLECT_LTO_WRAPPER=/usr/lib/gcc/x86_64-linux-gnu/11/lto-wrapper
OFFLOAD_TARGET_NAMES=nvptx-none:amdgcn-amdhsa
OFFLOAD_TARGET_DEFAULT=1
Target: x86_64-linux-gnu
Configured with: ../src/configure -v --with-pkgversion='Ubuntu 11.2.0-19ubuntu1' --with-bugurl=file:///usr/share/doc/gcc-11/README.Bugs --enable-languages=c,ada,c++,go,brig,d,fortran,objc,obj-c++,m2 --prefix=/usr --with-gcc-major-version-only --program-suffix=-11 --program-prefix=x86_64-linux-gnu- --enable-shared --enable-linker-build-id --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --libdir=/usr/lib --enable-nls --enable-bootstrap --enable-clocale=gnu --enable-libstdcxx-debug --enable-libstdcxx-time=yes --with-default-libstdcxx-abi=new --enable-gnu-unique-object --disable-vtable-verify --enable-plugin --enable-default-pie --with-system-zlib --enable-libphobos-checking=release --with-target-system-zlib=auto --enable-objc-gc=auto --enable-multiarch --disable-werror --enable-cet --with-arch-32=i686 --with-abi=m64 --with-multilib-list=m32,m64,mx32 --enable-multilib --with-tune=generic --enable-offload-targets=nvptx-none=/build/gcc-11-gBFGDP/gcc-11-11.2.0/debian/tmp-nvptx/usr,amdgcn-amdhsa=/build/gcc-11-gBFGDP/gcc-11-11.2.0/debian/tmp-gcn/usr --without-cuda-driver --enable-checking=release --build=x86_64-linux-gnu --host=x86_64-linux-gnu --target=x86_64-linux-gnu --with-build-config=bootstrap-lto-lean --enable-link-serialization=2
Thread model: posix
Supported LTO compression algorithms: zlib zstd
gcc version 11.2.0 (Ubuntu 11.2.0-19ubuntu1)
The text was updated successfully, but these errors were encountered: