Skip to content

feat: add perf walltime support #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ on:
jobs:
tests:
runs-on: ubuntu-latest
steps:
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
submodules: "recursive"

- name: Cache build
uses: actions/cache@v3
Expand All @@ -29,7 +31,7 @@ jobs:
make -j

- name: Run tests
run: |
run: |
cd core/build-tests
GTEST_OUTPUT=json:test-results/ ctest

Expand All @@ -54,6 +56,8 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
submodules: "recursive"

- name: Cache build
uses: actions/cache@v3
Expand All @@ -73,6 +77,8 @@ jobs:
- name: Run the benchmarks
uses: CodSpeedHQ/action@main
if: matrix.codspeed-mode != 'off'
env:
CODSPEED_PERF_ENABLED: true
with:
run: examples/google_benchmark_cmake/build/benchmark_example
token: ${{ secrets.CODSPEED_TOKEN }}
Expand All @@ -90,6 +96,8 @@ jobs:
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
with:
submodules: "recursive"

- name: Set up Bazel
uses: bazel-contrib/[email protected]
Expand All @@ -104,10 +112,12 @@ jobs:
- name: Build and run benchmarks
run: |
bazel build //examples/google_benchmark_bazel:my_benchmark --//core:codspeed_mode=${{ matrix.codspeed-mode }}

- name: Run the benchmarks
uses: CodSpeedHQ/action@main
if: matrix.codspeed-mode != 'off'
env:
CODSPEED_PERF_ENABLED: true
with:
run: bazel run //examples/google_benchmark_bazel:my_benchmark --//core:codspeed_mode=${{ matrix.codspeed-mode }}
token: ${{ secrets.CODSPEED_TOKEN }}
Expand All @@ -120,6 +130,8 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
submodules: "recursive"

- name: Cache build
uses: actions/cache@v3
Expand Down Expand Up @@ -148,6 +160,8 @@ jobs:
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
with:
submodules: "recursive"

- name: Set up Bazel
uses: bazel-contrib/[email protected]
Expand Down
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "core/instrument-hooks"]
path = core/instrument-hooks
url = https://github.com/CodSpeedHQ/instrument-hooks
17 changes: 17 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
exclude: '^(google_benchmark/.*|core/instrument-hooks/.*|.*/build/.*|build/.*|core/include/valgrind\.h|core/include/callgrind\.h)'
files: ^(core|examples)/.*$

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-added-large-files
- repo: https://github.com/cpp-linter/cpp-linter-hooks
rev: v0.6.1
hooks:
- id: clang-format
files: \.(cpp|cc|cxx|h|hpp)$
21 changes: 21 additions & 0 deletions core/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,26 @@ config_setting(
constraint_values = ["@platforms//os:windows"],
)

# Define the instrument-hooks library separately to apply warning suppressions
cc_library(
name = "instrument_hooks",
srcs = ["instrument-hooks/dist/core.c"],
hdrs = glob(["instrument-hooks/includes/*.h"]),
includes = ["instrument-hooks/includes"],
copts = select({
":windows": [
"/wd4101", # unreferenced local variable (equivalent to -Wno-unused-variable)
"/wd4189", # local variable is initialized but not referenced (equivalent to -Wno-unused-but-set-variable)
"/wd4100", # unreferenced formal parameter (equivalent to -Wno-unused-parameter)
],
"//conditions:default": [
"-Wno-unused-variable",
"-Wno-unused-parameter",
"-Wno-unused-but-set-variable",
],
}),
)

# Define the codspeed library
cc_library(
name = "codspeed",
Expand All @@ -25,6 +45,7 @@ cc_library(
":walltime_mode": ["CODSPEED_ENABLED", "CODSPEED_WALLTIME"],
"//conditions:default": [],
}),
deps = [":instrument_hooks"],
visibility = ["//visibility:public"],
)

Expand Down
36 changes: 34 additions & 2 deletions core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,44 @@ cmake_minimum_required(VERSION 3.10)

set(CODSPEED_VERSION 1.2.0)

project(codspeed VERSION ${CODSPEED_VERSION} LANGUAGES CXX)
project(codspeed VERSION ${CODSPEED_VERSION} LANGUAGES CXX C)

# Specify the C++ standard
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED True)

# Add the include directory
include_directories(include)
include_directories(instrument-hooks/includes)

# Add the library
add_library(
instrument_hooks
instrument-hooks/dist/core.c
)

# Suppress warnings for the instrument_hooks library
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
target_compile_options(
instrument_hooks
PRIVATE
-Wno-maybe-uninitialized
-Wno-unused-variable
-Wno-unused-parameter
-Wno-unused-but-set-variable
-Wno-type-limits
)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
target_compile_options(
instrument_hooks
PRIVATE
/wd4101 # unreferenced local variable (equivalent to -Wno-unused-variable)
/wd4189 # local variable is initialized but not referenced (equivalent to -Wno-unused-but-set-variable)
/wd4100 # unreferenced formal parameter (equivalent to -Wno-unused-parameter)
)
endif()

# Add the main library
add_library(
codspeed
src/codspeed.cpp
Expand All @@ -20,13 +48,17 @@ add_library(
src/workspace.cpp
)

# Link instrument_hooks to codspeed
target_link_libraries(codspeed PRIVATE instrument_hooks)

# Version
add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}")

# Specify the include directories for users of the library
target_include_directories(
codspeed
PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/instrument-hooks/includes>
)

# Disable valgrind compilation errors
Expand Down Expand Up @@ -116,7 +148,7 @@ install(
)

install(
TARGETS codspeed
TARGETS codspeed instrument_hooks
EXPORT codspeed-targets
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
Expand Down
45 changes: 39 additions & 6 deletions core/include/measurement.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,28 @@
#define MEASUREMENT_H

#include <string>
#ifdef _WIN32
#include <process.h>
#else
#include <unistd.h>
#endif

#ifdef CODSPEED_INSTRUMENTATION
#include "callgrind.h"
#endif

extern "C" {
#include "core.h"
}

inline InstrumentHooks* get_hooks() {
static InstrumentHooks* g_hooks = nullptr;
if (!g_hooks) {
g_hooks = instrument_hooks_init();
}
return g_hooks;
}

inline std::string get_version() {
#ifdef CODSPEED_VERSION
return {CODSPEED_VERSION};
Expand All @@ -16,29 +33,45 @@ inline std::string get_version() {
}

#ifdef CODSPEED_INSTRUMENTATION
inline bool measurement_is_instrumented() { return RUNNING_ON_VALGRIND; }
inline bool measurement_is_instrumented() {
return instrument_hooks_is_instrumented(get_hooks());
}

inline void measurement_set_metadata() {
std::string metadata = "Metadata: codspeed-cpp " + get_version();
CALLGRIND_DUMP_STATS_AT(metadata.c_str());
std::string version = get_version();
instrument_hooks_set_integration(get_hooks(), "codspeed-cpp",
version.c_str());
}

__attribute__((always_inline)) inline void measurement_start() {
// Keep the callgrind macros here, so that they are properly inlined.
// Otherwise, we have an additional function call overhead to the
// instrument-hooks library.
CALLGRIND_ZERO_STATS;
CALLGRIND_START_INSTRUMENTATION;

instrument_hooks_start_benchmark(get_hooks());
}

__attribute__((always_inline)) inline void measurement_stop(
const std::string &name) {
const std::string& name) {
CALLGRIND_STOP_INSTRUMENTATION;
CALLGRIND_DUMP_STATS_AT(name.c_str());

instrument_hooks_stop_benchmark(get_hooks());

#ifdef _WIN32
auto current_pid = _getpid();
#else
auto current_pid = getpid();
#endif
instrument_hooks_executed_benchmark(get_hooks(), current_pid, name.c_str());
};
#else
// Stub implementations for non-instrumentation builds
inline bool measurement_is_instrumented() { return false; }
inline void measurement_set_metadata() {}
inline void measurement_start() {}
inline void measurement_stop(const std::string &name) { (void)name; }
inline void measurement_stop(const std::string& name) { (void)name; }
#endif

#endif // MEASUREMENT_H
1 change: 1 addition & 0 deletions core/instrument-hooks
Submodule instrument-hooks added at b003e5
2 changes: 1 addition & 1 deletion core/src/utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ std::string safe_getenv(const char* var_name);

} // namespace codspeed

#endif // CODSPEED_UTILS_H
#endif // CODSPEED_UTILS_H
1 change: 1 addition & 0 deletions examples/google_benchmark_bazel/sleep_bench.hpp
8 changes: 4 additions & 4 deletions examples/google_benchmark_cmake/fixture_bench.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ BENCHMARK_TEMPLATE_F(MyTemplatedFixture, IntTest, int)(benchmark::State &st) {
for (auto _ : st) {
}
}
BENCHMARK_TEMPLATE_DEFINE_F(MyTemplatedFixture, DoubleTest,
double)(benchmark::State &st) {
BENCHMARK_TEMPLATE_DEFINE_F(MyTemplatedFixture, DoubleTest, double)
(benchmark::State &st) {
for (auto _ : st) {
}
}
Expand All @@ -56,8 +56,8 @@ BENCHMARK_REGISTER_F(MyTemplate1, TestA);

template <typename T, typename U>
class MyTemplate2 : public benchmark::Fixture {};
BENCHMARK_TEMPLATE2_DEFINE_F(MyTemplate2, TestB, int,
double)(benchmark::State &st) {
BENCHMARK_TEMPLATE2_DEFINE_F(MyTemplate2, TestB, int, double)
(benchmark::State &st) {
for (auto _ : st) {
}
}
Expand Down
1 change: 1 addition & 0 deletions examples/google_benchmark_cmake/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <cstring>

#include "fixture_bench.hpp"
#include "sleep_bench.hpp"
#include "template_bench.hpp"

template <class... Args>
Expand Down
62 changes: 62 additions & 0 deletions examples/google_benchmark_cmake/sleep_bench.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#pragma once

#include <benchmark/benchmark.h>

#include <chrono>
#include <thread>

static void BM_sleep_1ns(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::nanoseconds(1));
}
}
BENCHMARK(BM_sleep_1ns);

static void BM_sleep_100ns(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::nanoseconds(100));
}
}
BENCHMARK(BM_sleep_100ns);

static void BM_sleep_1us(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
}
BENCHMARK(BM_sleep_1us);

static void BM_sleep_100us(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::microseconds(100));
}
}
BENCHMARK(BM_sleep_100us);

static void BM_sleep_1ms(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
}
BENCHMARK(BM_sleep_1ms);

static void BM_sleep_10ms(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
BENCHMARK(BM_sleep_10ms);

static void BM_sleep_50ms(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
}
BENCHMARK(BM_sleep_50ms);

static void BM_sleep_100ms(benchmark::State& state) {
for (auto _ : state) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}
BENCHMARK(BM_sleep_100ms);
Loading
Loading