Skip to content

Commit

Permalink
Add C++ demo for VAD+non-streaming ASR (#1964)
Browse files Browse the repository at this point in the history
  • Loading branch information
@csukuangfj
csukuangfj authored Mar 7, 2025
1 parent 1e23282 commit 362ddf2
Show file tree
Hide file tree
Showing 6 changed files with 276 additions and 45 deletions.
1 change: 1 addition & 0 deletions cmake/cmake_extension.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def get_binaries():
"sherpa-onnx-online-websocket-server",
"sherpa-onnx-vad-microphone",
"sherpa-onnx-vad-microphone-offline-asr",
"sherpa-onnx-vad-with-offline-asr",
]

if enable_alsa():
Expand Down
5 changes: 5 additions & 0 deletions sherpa-onnx/csrc/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,10 @@ if(SHERPA_ONNX_ENABLE_PORTAUDIO AND SHERPA_ONNX_ENABLE_BINARY)
microphone.cc
)

add_executable(sherpa-onnx-vad-with-offline-asr
sherpa-onnx-vad-with-offline-asr.cc
)

add_executable(sherpa-onnx-vad-microphone-offline-asr
sherpa-onnx-vad-microphone-offline-asr.cc
microphone.cc
Expand All @@ -475,6 +479,7 @@ if(SHERPA_ONNX_ENABLE_PORTAUDIO AND SHERPA_ONNX_ENABLE_BINARY)
sherpa-onnx-microphone-offline-audio-tagging
sherpa-onnx-vad-microphone
sherpa-onnx-vad-microphone-offline-asr
sherpa-onnx-vad-with-offline-asr
)
if(SHERPA_ONNX_ENABLE_TTS)
list(APPEND exes
Expand Down
71 changes: 29 additions & 42 deletions sherpa-onnx/csrc/online-ebranchformer-transducer-model.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,8 @@ OnlineEbranchformerTransducerModel::OnlineEbranchformerTransducerModel(
}
}


void OnlineEbranchformerTransducerModel::InitEncoder(void *model_data,
size_t model_data_length) {
size_t model_data_length) {
encoder_sess_ = std::make_unique<Ort::Session>(
env_, model_data, model_data_length, encoder_sess_opts_);

Expand Down Expand Up @@ -153,9 +152,8 @@ void OnlineEbranchformerTransducerModel::InitEncoder(void *model_data,
}
}


void OnlineEbranchformerTransducerModel::InitDecoder(void *model_data,
size_t model_data_length) {
size_t model_data_length) {
decoder_sess_ = std::make_unique<Ort::Session>(
env_, model_data, model_data_length, decoder_sess_opts_);

Expand All @@ -180,7 +178,7 @@ void OnlineEbranchformerTransducerModel::InitDecoder(void *model_data,
}

void OnlineEbranchformerTransducerModel::InitJoiner(void *model_data,
size_t model_data_length) {
size_t model_data_length) {
joiner_sess_ = std::make_unique<Ort::Session>(
env_, model_data, model_data_length, joiner_sess_opts_);

Expand All @@ -200,7 +198,6 @@ void OnlineEbranchformerTransducerModel::InitJoiner(void *model_data,
}
}


std::vector<Ort::Value> OnlineEbranchformerTransducerModel::StackStates(
const std::vector<std::vector<Ort::Value>> &states) const {
int32_t batch_size = static_cast<int32_t>(states.size());
Expand All @@ -215,28 +212,28 @@ std::vector<Ort::Value> OnlineEbranchformerTransducerModel::StackStates(
ans.reserve(num_states);

for (int32_t i = 0; i != num_hidden_layers_; ++i) {
{ // cached_key
{ // cached_key
for (int32_t n = 0; n != batch_size; ++n) {
buf[n] = &states[n][4 * i];
}
auto v = Cat(allocator, buf, /* axis */ 0);
ans.push_back(std::move(v));
}
{ // cached_value
{ // cached_value
for (int32_t n = 0; n != batch_size; ++n) {
buf[n] = &states[n][4 * i + 1];
}
auto v = Cat(allocator, buf, 0);
ans.push_back(std::move(v));
}
{ // cached_conv
{ // cached_conv
for (int32_t n = 0; n != batch_size; ++n) {
buf[n] = &states[n][4 * i + 2];
}
auto v = Cat(allocator, buf, 0);
ans.push_back(std::move(v));
}
{ // cached_conv_fusion
{ // cached_conv_fusion
for (int32_t n = 0; n != batch_size; ++n) {
buf[n] = &states[n][4 * i + 3];
}
Expand All @@ -245,7 +242,7 @@ std::vector<Ort::Value> OnlineEbranchformerTransducerModel::StackStates(
}
}

{ // processed_lens
{ // processed_lens
for (int32_t n = 0; n != batch_size; ++n) {
buf[n] = &states[n][num_states - 1];
}
Expand All @@ -256,11 +253,9 @@ std::vector<Ort::Value> OnlineEbranchformerTransducerModel::StackStates(
return ans;
}


std::vector<std::vector<Ort::Value>>
OnlineEbranchformerTransducerModel::UnStackStates(
const std::vector<Ort::Value> &states) const {

assert(static_cast<int32_t>(states.size()) == num_hidden_layers_ * 4 + 1);

int32_t batch_size = states[0].GetTensorTypeAndShapeInfo().GetShape()[0];
Expand All @@ -272,31 +267,31 @@ OnlineEbranchformerTransducerModel::UnStackStates(
ans.resize(batch_size);

for (int32_t i = 0; i != num_hidden_layers_; ++i) {
{ // cached_key
{ // cached_key
auto v = Unbind(allocator, &states[i * 4], /* axis */ 0);
assert(static_cast<int32_t>(v.size()) == batch_size);

for (int32_t n = 0; n != batch_size; ++n) {
ans[n].push_back(std::move(v[n]));
}
}
{ // cached_value
{ // cached_value
auto v = Unbind(allocator, &states[i * 4 + 1], 0);
assert(static_cast<int32_t>(v.size()) == batch_size);

for (int32_t n = 0; n != batch_size; ++n) {
ans[n].push_back(std::move(v[n]));
}
}
{ // cached_conv
{ // cached_conv
auto v = Unbind(allocator, &states[i * 4 + 2], 0);
assert(static_cast<int32_t>(v.size()) == batch_size);

for (int32_t n = 0; n != batch_size; ++n) {
ans[n].push_back(std::move(v[n]));
}
}
{ // cached_conv_fusion
{ // cached_conv_fusion
auto v = Unbind(allocator, &states[i * 4 + 3], 0);
assert(static_cast<int32_t>(v.size()) == batch_size);

Expand All @@ -306,7 +301,7 @@ OnlineEbranchformerTransducerModel::UnStackStates(
}
}

{ // processed_lens
{ // processed_lens
auto v = Unbind<int64_t>(allocator, &states.back(), 0);
assert(static_cast<int32_t>(v.size()) == batch_size);

Expand All @@ -318,7 +313,6 @@ OnlineEbranchformerTransducerModel::UnStackStates(
return ans;
}


std::vector<Ort::Value>
OnlineEbranchformerTransducerModel::GetEncoderInitStates() {
std::vector<Ort::Value> ans;
Expand All @@ -332,40 +326,37 @@ OnlineEbranchformerTransducerModel::GetEncoderInitStates() {
int32_t channels_conv_fusion = 2 * hidden_size_;

for (int32_t i = 0; i != num_hidden_layers_; ++i) {
{ // cached_key_{i}
{ // cached_key_{i}
std::array<int64_t, 4> s{1, num_heads_, left_context_len_, head_dim_};
auto v =
Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
auto v = Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
Fill(&v, 0);
ans.push_back(std::move(v));
}

{ // cahced_value_{i}
{ // cahced_value_{i}
std::array<int64_t, 4> s{1, num_heads_, left_context_len_, head_dim_};
auto v =
Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
auto v = Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
Fill(&v, 0);
ans.push_back(std::move(v));
}

{ // cached_conv_{i}
{ // cached_conv_{i}
std::array<int64_t, 3> s{1, channels_conv, left_context_conv};
auto v =
Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
auto v = Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
Fill(&v, 0);
ans.push_back(std::move(v));
}

{ // cached_conv_fusion_{i}
std::array<int64_t, 3> s{1, channels_conv_fusion, left_context_conv_fusion};
auto v =
Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
{ // cached_conv_fusion_{i}
std::array<int64_t, 3> s{1, channels_conv_fusion,
left_context_conv_fusion};
auto v = Ort::Value::CreateTensor<float>(allocator_, s.data(), s.size());
Fill(&v, 0);
ans.push_back(std::move(v));
}
} // num_hidden_layers_

{ // processed_lens
{ // processed_lens
std::array<int64_t, 1> s{1};
auto v = Ort::Value::CreateTensor<int64_t>(allocator_, s.data(), s.size());
Fill<int64_t>(&v, 0);
Expand All @@ -375,11 +366,10 @@ OnlineEbranchformerTransducerModel::GetEncoderInitStates() {
return ans;
}


std::pair<Ort::Value, std::vector<Ort::Value>>
OnlineEbranchformerTransducerModel::RunEncoder(Ort::Value features,
std::vector<Ort::Value> states,
Ort::Value /* processed_frames */) {
OnlineEbranchformerTransducerModel::RunEncoder(
Ort::Value features, std::vector<Ort::Value> states,
Ort::Value /* processed_frames */) {
std::vector<Ort::Value> encoder_inputs;
encoder_inputs.reserve(1 + states.size());

Expand All @@ -402,7 +392,6 @@ OnlineEbranchformerTransducerModel::RunEncoder(Ort::Value features,
return {std::move(encoder_out[0]), std::move(next_states)};
}


Ort::Value OnlineEbranchformerTransducerModel::RunDecoder(
Ort::Value decoder_input) {
auto decoder_out = decoder_sess_->Run(
Expand All @@ -411,9 +400,8 @@ Ort::Value OnlineEbranchformerTransducerModel::RunDecoder(
return std::move(decoder_out[0]);
}


Ort::Value OnlineEbranchformerTransducerModel::RunJoiner(Ort::Value encoder_out,
Ort::Value decoder_out) {
Ort::Value OnlineEbranchformerTransducerModel::RunJoiner(
Ort::Value encoder_out, Ort::Value decoder_out) {
std::array<Ort::Value, 2> joiner_input = {std::move(encoder_out),
std::move(decoder_out)};
auto logit =
Expand All @@ -424,7 +412,6 @@ Ort::Value OnlineEbranchformerTransducerModel::RunJoiner(Ort::Value encoder_out,
return std::move(logit[0]);
}


#if __ANDROID_API__ >= 9
template OnlineEbranchformerTransducerModel::OnlineEbranchformerTransducerModel(
AAssetManager *mgr, const OnlineModelConfig &config);
Expand Down
2 changes: 1 addition & 1 deletion sherpa-onnx/csrc/online-ebranchformer-transducer-model.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class OnlineEbranchformerTransducerModel : public OnlineTransducerModel {

template <typename Manager>
OnlineEbranchformerTransducerModel(Manager *mgr,
const OnlineModelConfig &config);
const OnlineModelConfig &config);

std::vector<Ort::Value> StackStates(
const std::vector<std::vector<Ort::Value>> &states) const override;
Expand Down
4 changes: 2 additions & 2 deletions sherpa-onnx/csrc/sherpa-onnx-offline.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,10 @@ for a list of pre-trained models to download.
std::vector<sherpa_onnx::OfflineStream *> ss_pointers;
float duration = 0;
for (int32_t i = 1; i <= po.NumArgs(); ++i) {
const std::string wav_filename = po.GetArg(i);
std::string wav_filename = po.GetArg(i);
int32_t sampling_rate = -1;
bool is_ok = false;
const std::vector<float> samples =
std::vector<float> samples =
sherpa_onnx::ReadWave(wav_filename, &sampling_rate, &is_ok);
if (!is_ok) {
fprintf(stderr, "Failed to read '%s'\n", wav_filename.c_str());
Expand Down
Loading

0 comments on commit 362ddf2

Please sign in to comment.