diff --git a/_data/authors.yml b/_data/authors.yml index 937e0f2a..9e26c10a 100644 --- a/_data/authors.yml +++ b/_data/authors.yml @@ -1,272 +1,353 @@ # Author Details francois: - name: François Baldassari - email: francois@memfault.com - web: https://twitter.com/baldassarifr - image: /img/author/francois.jpg - blurb: has worked on the embedded software teams at Sun, Pebble, and Oculus. He is currently the CEO of Memfault. - twitter: baldassarifr - linkedin: https://www.linkedin.com/in/francois-baldassari - github: franc0is + name: François Baldassari + email: francois@memfault.com + web: https://twitter.com/baldassarifr + image: /img/author/francois.jpg + blurb: + has worked on the embedded software teams at Sun, Pebble, and Oculus. He is + currently the CEO of Memfault. + twitter: baldassarifr + linkedin: https://www.linkedin.com/in/francois-baldassari + github: franc0is chris: - name: Chris Coleman - email: chris@memfault.com - web: https://www.linkedin.com/in/christopher-coleman-812aa06b/ - image: /img/author/chris.jpg - blurb: is a founder and CTO at Memfault. Prior to founding Memfault, Chris worked on the embedded software teams at Sun, Pebble, and Fitbit. - github: chrisc11 - linkedin: https://www.linkedin.com/in/christopher-coleman-812aa06b/ + name: Chris Coleman + email: chris@memfault.com + web: https://www.linkedin.com/in/christopher-coleman-812aa06b/ + image: /img/author/chris.jpg + blurb: + is a founder and CTO at Memfault. Prior to founding Memfault, Chris worked + on the embedded software teams at Sun, Pebble, and Fitbit. + github: chrisc11 + linkedin: https://www.linkedin.com/in/christopher-coleman-812aa06b/ tyler: - name: Tyler Hoffman - email: tyler@memfault.com - web: https://www.linkedin.com/in/tyhoff/ - image: /img/author/tyler.jpg - blurb: has worked on the embedded software teams at Pebble and Fitbit. He is now a founder at Memfault. - twitter: ty_hoff - linkedin: https://www.linkedin.com/in/tyhoff/ - github: tyhoff + name: Tyler Hoffman + email: tyler@memfault.com + web: https://www.linkedin.com/in/tyhoff/ + image: /img/author/tyler.jpg + blurb: + has worked on the embedded software teams at Pebble and Fitbit. He is now a + founder at Memfault. + twitter: ty_hoff + linkedin: https://www.linkedin.com/in/tyhoff/ + github: tyhoff mafaneh: - name: Mohammad Afaneh - email: mohammad@novelbits.io - web: https://www.novelbits.io - image: /img/author/mafaneh.jpg - blurb: is the founder and CEO of Novel Bits, a company that focuses on providing comprehensive tutorials, resources, and online training for Bluetooth developers. - twitter: m_afaneh - linkedin: https://www.linkedin.com/in/mafaneh/ - github: mafaneh + name: Mohammad Afaneh + email: mohammad@novelbits.io + web: https://www.novelbits.io + image: /img/author/mafaneh.jpg + blurb: + is the founder and CEO of Novel + Bits, a company that focuses on providing comprehensive tutorials, + resources, and online training for Bluetooth developers. + twitter: m_afaneh + linkedin: https://www.linkedin.com/in/mafaneh/ + github: mafaneh cyril: - name: Cyril Fougeray - email: cyril.fougeray@gmail.com - web: http://www.cyrilfougeray.com - linkedin: https://www.linkedin.com/in/cyrilfougeray/ - twitter: cyrilfougeray - github: fouge - image: /img/author/cyril.jpg - blurb: is a freelance embedded software engineer you can hire through that page. Previously he worked on embedded software at Equisense and Spire. + name: Cyril Fougeray + email: cyril.fougeray@gmail.com + web: http://www.cyrilfougeray.com + linkedin: https://www.linkedin.com/in/cyrilfougeray/ + twitter: cyrilfougeray + github: fouge + image: /img/author/cyril.jpg + blurb: + is a freelance embedded software engineer you can hire through that page. Previously he + worked on embedded software at Equisense and Spire. james: - name: James Munns - email: james.munns@ferrous-systems.com - web: https://ferrous-systems.com - twitter: bitshiftmask - image: /img/author/james.jpg - github: jamesmunns - linkedin: https://www.linkedin.com/in/james-munns-8a42b429/ - blurb: has worked on embedded systems from Safety Critical Avionics to rapidly prototyped IoT solutions. He is a Founder and Managing Director of Ferrous Systems. + name: James Munns + email: james.munns@ferrous-systems.com + web: https://ferrous-systems.com + twitter: bitshiftmask + image: /img/author/james.jpg + github: jamesmunns + linkedin: https://www.linkedin.com/in/james-munns-8a42b429/ + blurb: + has worked on embedded systems from Safety Critical Avionics to rapidly + prototyped IoT solutions. He is a Founder and Managing Director of Ferrous Systems. rameen: - name: Rameen Aryanpur - email: raryanpur@andium.com - web: https://www.linkedin.com/in/rameenaryanpur/ - image: /img/author/rameen-headshot.png - twitter: raryanpur - linked: https://www.linkedin.com/in/rameenaryanpur/ - blurb: leads the engineering team at Andium. He previously worked as a software engineer in robotics and distributed systems at Boston Engineering, and was co-founder and CTO of a connected device startup. + name: Rameen Aryanpur + email: raryanpur@andium.com + web: https://www.linkedin.com/in/rameenaryanpur/ + image: /img/author/rameen-headshot.png + twitter: raryanpur + linked: https://www.linkedin.com/in/rameenaryanpur/ + blurb: + leads the engineering team at Andium. He previously worked as a software + engineer in robotics and distributed systems at Boston Engineering, and was + co-founder and CTO of a connected device startup. noah: - name: Noah Pendleton - email: noah@memfault.com - web: https://www.linkedin.com/in/noahpendleton/ - linkedin: https://www.linkedin.com/in/noahpendleton/ - image: /img/author/noah.jpg - twitter: nlpendleton - github: noahp - blurb: is an embedded software engineer at Memfault. Noah previously worked on embedded software teams at Fitbit and Markforged + name: Noah Pendleton + email: noah@memfault.com + web: https://www.linkedin.com/in/noahpendleton/ + linkedin: https://www.linkedin.com/in/noahpendleton/ + image: /img/author/noah.jpg + twitter: nlpendleton + github: noahp + blurb: + is an embedded software engineer at Memfault. Noah previously worked on + embedded software teams at Fitbit and Markforged shiva: - name: Shiva Rajagopal - email: shiva.rajagopal@outlook.com - web: https://about.me/shiva.rajagopal - linkedin: https://www.linkedin.com/in/shivarajagopal/ - image: /img/author/shiva.jpg - github: shivarajagopal - blurb: is an embedded software engineer at Fitbit. + name: Shiva Rajagopal + email: shiva.rajagopal@outlook.com + web: https://about.me/shiva.rajagopal + linkedin: https://www.linkedin.com/in/shivarajagopal/ + image: /img/author/shiva.jpg + github: shivarajagopal + blurb: is an embedded software engineer at Fitbit. jaredwolff: - name: Jared Wolff - email: hello@jaredwolff.com - web: https://www.jaredwolff.com - linkedin: https://www.linkedin.com/in/jaredwolff/ - image: /img/author/jaredwolff.jpg - twitter: jaredwolff - github: jaredwolff - blurb: Hardware and firmware enthusiast. Proud father of the nRF9160 Feather. + name: Jared Wolff + email: hello@jaredwolff.com + web: https://www.jaredwolff.com + linkedin: https://www.linkedin.com/in/jaredwolff/ + image: /img/author/jaredwolff.jpg + twitter: jaredwolff + github: jaredwolff + blurb: + Hardware and firmware enthusiast. Proud father of the nRF9160 Feather. amundas: - name: Amund Askeland - email: amund.ask@gmail.com - github: amundas - linkedin: https://www.linkedin.com/in/amund-askeland-9a2545149/ - blurb: is a hardware and software enthusiast currently working on a PhD in embedded security. - image: /img/author/amund.png + name: Amund Askeland + email: amund.ask@gmail.com + github: amundas + linkedin: https://www.linkedin.com/in/amund-askeland-9a2545149/ + blurb: + is a hardware and software enthusiast currently working on a PhD in embedded + security. + image: /img/author/amund.png ael-mess: - name: Amine El Messaoudi - email: amine.elmessaoudi@icloud.com - github: ael-mess - linkedin: https://www.linkedin.com/in/ael-mess/ - blurb: is an embedded software and firmware engineer, passionate about low-level programming. - image: /img/author/ael-mess.jpg + name: Amine El Messaoudi + email: amine.elmessaoudi@icloud.com + github: ael-mess + linkedin: https://www.linkedin.com/in/ael-mess/ + blurb: + is an embedded software and firmware engineer, passionate about low-level + programming. + image: /img/author/ael-mess.jpg erikfogg: - name: Erik Fogg - blurb: is the Chief Operating Officer at ProdPerfect, which is an autonomous E2E regression testing solution that leverages data from live user behavior data. - linkedin: https://www.linkedin.com/in/erikfogg/ - image: /img/author/erikfogg.jpeg + name: Erik Fogg + blurb: + is the Chief Operating Officer at ProdPerfect, which is an autonomous E2E + regression testing solution that leverages data from live user behavior + data. + linkedin: https://www.linkedin.com/in/erikfogg/ + image: /img/author/erikfogg.jpeg nickmiller: - name: Nick Miller - blurb: is a principal embedded software engineer at SpinDance, an IoT solutions company. - linkedin: https://www.linkedin.com/in/nicholas-miller-666a37b7/ - github: ncmiller - image: /img/author/nickmiller.png + name: Nick Miller + blurb: + is a principal embedded software engineer at SpinDance, an IoT solutions company. + linkedin: https://www.linkedin.com/in/nicholas-miller-666a37b7/ + github: ncmiller + image: /img/author/nickmiller.png thomas: - name: Thomas Höhenleitner - blurb: has developed hardware devices and moved to embedded software development. - github: rokath - image: /img/author/thomas.jpg + name: Thomas Höhenleitner + blurb: + has developed hardware devices and moved to embedded software development. + github: rokath + image: /img/author/thomas.jpg nash: - name: Nash Reilly - blurb: works as an independent EE consultant and runs RTLjobs.com, a job site for FPGA and RTL designers. He has previously written firmware for Sonos, Root.ai, and AppHarvest. - github: cushychicken - twitter: cushyChicken - linkedin: https://www.linkedin.com/in/nash-reilly-55284a33/ - image: /img/author/nash.png + name: Nash Reilly + blurb: + works as an independent EE consultant and runs RTLjobs.com, a job site + for FPGA and RTL designers. He has previously written firmware for Sonos, + Root.ai, and AppHarvest. + github: cushychicken + twitter: cushyChicken + linkedin: https://www.linkedin.com/in/nash-reilly-55284a33/ + image: /img/author/nash.png lars: - name: Lars Pötter - blurb: is a freelance embedded software engineer, has worked on 8, 16 and 32 bit architectures and in safety critic projects. Creator of chipselect.org. Email me at interrupt@ing-poetter.de - email: interrupt@ing-poetter.de - web: https://www.ing-poetter.de/ - linkedin: https://www.linkedin.com/in/lars-p%C3%B6tter-32b42960 + name: Lars Pötter + blurb: + is a freelance embedded software engineer, has worked on 8, 16 and 32 bit + architectures and in safety critic projects. Creator of chipselect.org. + Email me at interrupt@ing-poetter.de + email: interrupt@ing-poetter.de + web: https://www.ing-poetter.de/ + linkedin: https://www.linkedin.com/in/lars-p%C3%B6tter-32b42960 veverak: - name: Jan Koniarik - github: koniarik - linkedin: https://www.linkedin.com/in/jan-koniarik-21b9723b/ - twitter: https://twitter.com/SquirrelCZE - blurb: -- robotics software engineer, keen interest in embedded, teaching assistant at university + name: Jan Koniarik + github: koniarik + linkedin: https://www.linkedin.com/in/jan-koniarik-21b9723b/ + twitter: https://twitter.com/SquirrelCZE + blurb: + -- robotics software engineer, keen interest in embedded, teaching assistant + at university donatien: - name: Donatien Garnier - email: donatien.garnier@blecon.net - image: /img/author/donatien.png - blurb: is a co-founder of Blecon. He was previously tech lead for Arm Mbed OS, and founder of an IoT startup. - twitter: donatiengarnier - linkedin: https://www.linkedin.com/in/donatiengarnier/ - github: donatieng + name: Donatien Garnier + email: donatien.garnier@blecon.net + image: /img/author/donatien.png + blurb: + is a co-founder of Blecon. He was + previously tech lead for Arm Mbed OS, and founder of an IoT startup. + twitter: donatiengarnier + linkedin: https://www.linkedin.com/in/donatiengarnier/ + github: donatieng stawiski: - name: Mikolaj Stawiski - blurb: is a passionate software engineer coming from awesome electronics and hardware worlds. - linkedin: https://www.linkedin.com/in/mstawiski/ - github: stawiski - image: /img/author/stawiski.png + name: Mikolaj Stawiski + blurb: + is a passionate software engineer coming from awesome electronics and + hardware worlds. + linkedin: https://www.linkedin.com/in/mstawiski/ + github: stawiski + image: /img/author/stawiski.png mab: - name: Mohammed Billoo - blurb: is an embedded Linux consultant with over 14 years of experience in embedded software across a multitude of domains, including Defense, Space, and Commercial. He is an avid blogger and contributor to the open-source community. - linkedin: https://www.linkedin.com/in/mab-embedded/ - github: mabembedded - image: /img/author/mab.jpeg + name: Mohammed Billoo + blurb: + is an embedded Linux consultant with over 14 years of experience in embedded + software across a multitude of domains, including Defense, Space, and + Commercial. He is an avid blogger and contributor to the open-source + community. + linkedin: https://www.linkedin.com/in/mab-embedded/ + github: mabembedded + image: /img/author/mab.jpeg jonkurtz: - name: Jon Kurtz - email: jon.kurtz.uf@renesas.com - image: /img/author/jonkurtz.jpg - linkedin: https://www.linkedin.com/in/jonathankurtz1 - blurb: is an FAE Connectivity manager at Renesas. + name: Jon Kurtz + email: jon.kurtz.uf@renesas.com + image: /img/author/jonkurtz.jpg + linkedin: https://www.linkedin.com/in/jonathankurtz1 + blurb: is an FAE Connectivity manager at Renesas. tsarlandie: - name: Thomas Sarlandie - github: sarfata - image: /img/author/tsarlandie.jpg - blurb: is the Linux Tech Lead at Memfault. + name: Thomas Sarlandie + github: sarfata + image: /img/author/tsarlandie.jpg + blurb: is the Linux Tech Lead at Memfault. caglayan: - name: Çağlayan Dökme - blurb: is a young embedded software engineer who believes in the power of knowledge sharing. - email: caglayandokme@gmail.com - linkedin: https://www.linkedin.com/in/caglayandokme/ - github: CaglayanDokme + name: Çağlayan Dökme + blurb: + is a young embedded software engineer who believes in the power of knowledge + sharing. + email: caglayandokme@gmail.com + linkedin: https://www.linkedin.com/in/caglayandokme/ + github: CaglayanDokme ericj: - name: Eric Johnson - email: eric@memfault.com - github: ejohnso49 - web: https://www.linkedin.com/in/ejohnso49 - linkedin: https://www.linkedin.com/in/ejohnso49 - image: /img/author/ejohnso49.png - blurb: is a Firmware Solutions Engineer at Memfault. Eric previously worked on embedded software teams at Walgreens Health and Athos + name: Eric Johnson + email: eric@memfault.com + github: ejohnso49 + web: https://www.linkedin.com/in/ejohnso49 + linkedin: https://www.linkedin.com/in/ejohnso49 + image: /img/author/ejohnso49.png + blurb: + is a Firmware Solutions Engineer at Memfault. Eric previously worked on + embedded software teams at Walgreens Health and Athos blaise: - name: Blaise Lengrand - github: blaizard - image: /img/author/blaise.jpg - linkedin: https://www.linkedin.com/in/blaiselengrand - blurb: is a C++ software engineer with an insatiable passion for all things embedded and robotics! + name: Blaise Lengrand + github: blaizard + image: /img/author/blaise.jpg + linkedin: https://www.linkedin.com/in/blaiselengrand + blurb: + is a C++ software engineer with an insatiable passion for all things + embedded and robotics! mabe: - name: Markus Becker - blurb: is a Software Engineer for IoT and wireless topics, currently focussing on professional lighting equipment. - linkedin: https://www.linkedin.com/in/mabcomnets/ - github: markus-becker-tridonic-com - image: /img/author/mabe.jpg + name: Markus Becker + blurb: + is a Software Engineer for IoT and wireless topics, currently focussing on + professional lighting equipment. + linkedin: https://www.linkedin.com/in/mabcomnets/ + github: markus-becker-tridonic-com + image: /img/author/mabe.jpg blake: - name: Blake Hildebrand - blurb: is an embedded software engineer at Memfault. Blake previously worked on embedded software teams at Garmin and Amazon Robotics. - linkedin: https://www.linkedin.com/in/blake-a-hildebrand/ - github: bahildebrand - image: /img/author/blake.jpg + name: Blake Hildebrand + blurb: + is an embedded software engineer at Memfault. Blake previously worked on + embedded software teams at Garmin and Amazon Robotics. + linkedin: https://www.linkedin.com/in/blake-a-hildebrand/ + github: bahildebrand + image: /img/author/blake.jpg lampacher: - name: Martin Lampacher - image: /img/author/lampacher.jpg - linkedin: https://www.linkedin.com/in/martin-lampacher-4b532984 - blurb: is a Freelance Embedded Engineer based in the Italian Alps. - github: lmapii + name: Martin Lampacher + image: /img/author/lampacher.jpg + linkedin: https://www.linkedin.com/in/martin-lampacher-4b532984 + blurb: is a Freelance Embedded Engineer based in the Italian Alps. + github: lmapii chayes: - name: Chris Hayes - blurb: is an Android Solutions Engineer at Memfault. Chris previously worked at Square as an Embedded Android Engineer. - linkedin: https://www.linkedin.com/in/chrishayes6/ - github: phoenixuprising + name: Chris Hayes + blurb: + is an Android Solutions Engineer at Memfault. Chris previously worked at + Square as an Embedded Android Engineer. + linkedin: https://www.linkedin.com/in/chrishayes6/ + github: phoenixuprising guille: - name: Guillermo Garcia - web: https://embedded-dbl.com/ - blurb: is an embedded software engineer who likes IoT projects. - linkedin: https://www.linkedin.com/in/guillermo-garcia-ch%C3%A1vez-1126951ab/ - github: GuillerGC + name: Guillermo Garcia + web: https://embedded-dbl.com/ + blurb: is an embedded software engineer who likes IoT projects. + linkedin: https://www.linkedin.com/in/guillermo-garcia-ch%C3%A1vez-1126951ab/ + github: GuillerGC dangross: - name: Dan Gross - blurb: is passionate about reducing friction for developers. As a developer, Dan has worked on cloud and embedded systems for Sun, HP, Oracle, Samsung, and AWS. - image: /img/author/dangross.jpg - linkedin: https://www.linkedin.com/in/grossdan/ + name: Dan Gross + blurb: + is passionate about reducing friction for developers. As a developer, Dan + has worked on cloud and embedded systems for Sun, HP, Oracle, Samsung, and + AWS. + image: /img/author/dangross.jpg + linkedin: https://www.linkedin.com/in/grossdan/ ramanf: - name: Raman Filipau - github: romanf-dev - linkedin: https://www.linkedin.com/in/romanf-engineer/ - blurb: -- firmware engineer. Previously worked on industrial automation and OS/RTOS design research. + name: Raman Filipau + github: romanf-dev + linkedin: https://www.linkedin.com/in/romanf-engineer/ + blurb: + -- firmware engineer. Previously worked on industrial automation and OS/RTOS + design research. evgeny: - name: Evgeny Chormonov - web: https://ser-mk.github.io/ - blurb: is a hardware and software engineer - linkedin: https://linkedin.com/in/echormonov - github: ser-mk - image: /img/author/evgeny.png + name: Evgeny Chormonov + web: https://ser-mk.github.io/ + blurb: is a hardware and software engineer + linkedin: https://linkedin.com/in/echormonov + github: ser-mk + image: /img/author/evgeny.png piotrw: - name: Piotr Wasilewski - image: /img/author/piotrw.jpg - github: klonyyy - linkedin: https://www.linkedin.com/in/piotrek-wasilewski-9b63b198/ - blurb: -- embedded software engineer specializing in motor control applications - web: https://pwwprojects.blogspot.com + name: Piotr Wasilewski + image: /img/author/piotrw.jpg + github: klonyyy + linkedin: https://www.linkedin.com/in/piotrek-wasilewski-9b63b198/ + blurb: + -- embedded software engineer specializing in motor control applications + web: https://pwwprojects.blogspot.com zamuhrishka: - name: Aliaksandr Kavalchuk - image: /img/author/zamuhrishka.jpeg - github: Zamuhrishka - linkedin: https://www.linkedin.com/in/aleksander-kovalchuk-77030489/ - blurb: is a firmware engineer with over 10 years of experience. - web: https://medium.com/@aliaksandr.kavalchuk + name: Aliaksandr Kavalchuk + image: /img/author/zamuhrishka.jpeg + github: Zamuhrishka + linkedin: https://www.linkedin.com/in/aleksander-kovalchuk-77030489/ + blurb: is a firmware engineer with over 10 years of experience. + web: https://medium.com/@aliaksandr.kavalchuk gminn: - name: Gillian Minnehan - image: /img/author/gminn.jpeg - github: gminn - linkedin: https://www.linkedin.com/in/gillian-minnehan/ - blurb: is a firmware solutions architect at Memfault. Gillian previously worked on embedded software teams in space and defense. + name: Gillian Minnehan + image: /img/author/gminn.jpeg + github: gminn + linkedin: https://www.linkedin.com/in/gillian-minnehan/ + blurb: + is a firmware solutions architect at Memfault. Gillian previously worked on + embedded software teams in space and defense. victorlai: - name: Victor Lai - image: /img/author/victorlai.png - github: blisse - linkedin: https://www.linkedin.com/in/laivictor/ - blurb: is an Android solutions architect at Memfault. + name: Victor Lai + image: /img/author/victorlai.png + github: blisse + linkedin: https://www.linkedin.com/in/laivictor/ + blurb: is an Android solutions architect at Memfault. chrismerck: - name: Chris Merck - image: /img/author/chrismerck.jpg - github: chrismerck - linkedin: https://www.linkedin.com/in/christopher-merck-1987747a/ - blurb: is Co-Founder and CTO at Bond. + name: Chris Merck + image: /img/author/chrismerck.jpg + github: chrismerck + linkedin: https://www.linkedin.com/in/christopher-merck-1987747a/ + blurb: is Co-Founder and CTO at Bond. mschulte: - name: Mark Schulte - image: /img/author/mschulte.jpg - github: schultetwin - linkedin: https://www.linkedin.com/in/schulte-mark - blurb: is a firmware engineer trying to firmware \ No newline at end of file + name: Mark Schulte + image: /img/author/mschulte.jpg + github: schultetwin + linkedin: https://www.linkedin.com/in/schulte-mark + blurb: is a firmware engineer trying to firmware +bertschiettecatte: + name: Bert Schiettecatte + web: http://www.bertschiettecatte.com/ + image: /img/author/bertschiettecatte.png + linkedin: https://www.linkedin.com/in/bertschiettecatte/ + blurb: + is the founder of Noisetron LLC, a + software & hardware engineering and expert witness consulting practice, and + the founder of Percussa, a pro-audio + consumer electronics company. diff --git a/_posts/2024-08-29-memory-debugging.md b/_posts/2024-08-29-memory-debugging.md new file mode 100644 index 00000000..eef17fc5 --- /dev/null +++ b/_posts/2024-08-29-memory-debugging.md @@ -0,0 +1,218 @@ +--- +title: How Memory Usage Patterns Can Derail Real-time Performance +description: + A story about memory usage causing real-time performance issues in an audio + system. +author: bertschiettecatte +--- + + + +In this article, we will learn how memory usage patterns can affect the +real-time performance of an embedded application, drawing from a recent +experience tracing an audio DSP application running on an embedded Linux +platform. First, I will introduce the product in question and the real-time +audio software I developed for it. Then, I’ll describe the issues I encountered +with audio callbacks and the strategy I followed to determine the cause of the +issues, ending with my solution and lessons learned. + + + +{% include newsletter.html %} + +{% include toc.html %} + +## Background + +As the founder of Percussa, I have led the software and hardware engineering of +our pro-audio consumer electronics products for nearly 20 years. The first +product I developed was the +[AudioCubes](https://www.percussa.com/what-are-audiocubes/), a human-computer +interface, consisting of multiple smart wireless objects capable of detecting +each other’s location, orientation, and distance. The AudioCubes are a bare +metal DSP platform and do not run an off-the-shelf operating system. + +Subsequently, I developed the +[Percussa Super Signal Processor (SSP)](https://www.percussa.com/super-signal-processor/), +a multichannel audio DSP platform, which was funded via Kickstarter for 314%. +The SSP is used by professional studio musicians and sound designers worldwide +and is a eurorack module (eurorack is a modular synthesizer standard). It is +installed into end users’ synthesizer racks or in recording studios. It offers a +library of DSP building blocks that can be chained together to create various +signal-processing chains for synthesizing new sounds or processing existing +sounds. + +The SSP is a Linux-based platform based on a Rockchip SoC. In addition to +developing the hardware and bringing up the platform, I also developed the +real-time audio DSP software for the SSP. To deliver a great customer +experience, it was important to get consistent audio DSP performance from the +platform, avoiding the clicks and pops in the audio that can be caused by not +consistently meeting audio processing deadlines (i.e. completing audio callbacks +in time). Additionally, to fully benefit from the processing power of the +platform, DSP processing needs to be parallelized over multiple CPU cores (the +SoC in the SSP is a quad-core processor). + +

+ Image of Percussa device +

+ +The audio DSP software I developed originally ran on Windows and Mac OSX +platforms using a popular C++ application framework for audio applications. My +strategy was to port the software to the new audio-embedded Linux platform (the +SSP). + +This went fairly well, with most of my time spent designing and implementing +user interface code, menu systems, and code to parallelize DSP code across CPU +cores. My goal on my embedded Linux platform was to use the mainline kernel tree +to reduce maintenance costs. + +## Audio Issue Diagnosis + +As I started testing and debugging my application, I kept running into issues +where the audio output was regularly interrupted, resulting in an audible click. +What could possibly be wrong with my code that might cause this? I started +thinking about the reasons my audio callback could take too long to return, the +locks in my code, and whether the audio callback could be pre-empted by the +operating system, causing it to return too late and miss its real-time +“deadline.” + +Eventually, I decided to use a profiler to figure out what was going on and +turned to the excellent [Tracy Profiler](https://github.com/wolfpld/tracy), +which is open source and free. Tracy is easy to integrate with your code, +allowing you to define different zones for tracing and profiling. The Tracy +profiling GUI application can connect over the network with your embedded +platform and receive trace information in real time. This helps speed up +development and debugging. Tracy also comes with a command-line capture tool you +can use to capture the trace directly to disk instead of sending it over the +network to your workstation. + +I typically do not enable Tracy in release builds shipped to customers and only +use it when I’m trying to find a problem with the real-time behavior of my +software. I use compiler switches (`-D`) to enable Tracy, as described in the +[Tracy manual (PDF warning)](https://github.com/wolfpld/tracy/releases/latest/download/tracy.pdf), +which also describes how to control the amount of data collected. This is +important, as you do not want to generate so much tracing data that it starts to +get in the way of determining the real problem at hand. + +With the help of Tracy, I discovered that my audio callback was occasionally +taking too long to return. This happened often enough that it was a real +problem. + +

+ Image of Tracy (trace 1) +

+ +At the top of the screen in Tracy, you can see which CPU cores the threads are +being scheduled on (note the four “lanes” in the above screenshot, since the +software is running on a quad-core CPU). At the bottom of the screen, you can +see the different nested zones that I defined in my source code, with Tracy +measuring the time spent in each. The zones allow you to understand which areas +of your code are the most problematic and need review. + +You can see in the above screenshot an instance when the ALSA callback takes +much longer than normal. What was interesting was that the issue always seemed +to be connected to a memory-related function call. Right above the zones at the +bottom, you can see a series of blue dots. When hovering over them, these dots +showed the same memory-related function repeatedly, indicating that the CPU was +spending an unusually long time in this function. Additionally, the yellow graph +at the bottom shows memory usage constantly increasing or decreasing, resulting +in the zig-zag pattern being plotted. + +However, I had paid close attention to the code in my audio callback and DSP +worker threads and I was fairly confident I was not doing any kind of dynamic +memory allocation in my audio or DSP threads. + +As I dug deeper, I noticed that my audio thread was being pre-empted by kernel +threads. + +

+ Image of Tracy (trace 2) +

+ +While I don't have an exact copy of the pre-emption from this instance, I have +an example screenshot above which is very similar. In this case, you can see +that the ALSA thread is being pre-empted with a memory-related function call +appearing at the bottom of the screen shortly before the scheduling of kernel +threads. + +> Note: Tracy can sample Linux kernel call stacks in addition to your +> application call stacks to give an even deeper look at what the kernel is +> doing. Refer to the Tracy manual to understand the requirements for this. + +Eventually, I started examining the memory usage behavior of my application and +discovered that the third-party application framework I was using had its own +memory management classes. Under the hood, `malloc()` was being called to +allocate blocks of very small sizes -- some as small as 4 bytes! These classes +were used all over the code base in the framework -- and thus in my application, +which was based on the framework. I had unwittingly invited dynamic memory +allocation in places due to the way the framework was architected. + +Because of the frequency with which these calls happened, it resulted in memory +fragmentation, which explained why some memory-related function calls took so +long to return. Fragmentation can result in memory-related function calls taking +much longer than normal and/or the Linux kernel pre-empting your thread(s) while +it performs its cleanup. + +By analyzing the call stacks, combined with callbacks taking unusually long to +return, excessive time spent in memory-related functions, and the scheduling of +kernel threads, you can get an idea of whether your application’s memory usage +patterns might be causing issues. + +In my particular case, my application’s memory usage patterns caused my +callbacks and other code to take too long to return, resulting in missed +real-time deadlines and audio clicks. + +## Solution + +To solve this problem, I implemented my own memory allocator and re-implemented +`malloc()` and friends to use my allocator. At a high level, my allocator +pre-allocates one large block of memory using `mmap()` on startup and manages +free/used lists of smaller blocks. Calls to `malloc()` and friends then result +in allocations/deallocations using my allocator. This approach addresses the +above fragmentation issues and prevents memory-related function calls from +taking too long or varying unpredictably. + +I considered using off-the-shelf allocators, but this experience made me a bit +paranoid. I decided that implementing one myself was a good exercise anyway so I +could be aware of its behavior under the hood. By implementing my own allocator +and overriding `malloc()` and friends to use my allocator, I was able to collect +and display allocation statistics and understand how my application uses memory +as I was debugging and testing it. + +I also reviewed my code and re-architected parts of it to pre-allocate pools of +objects on startup, which are then re-used as much as possible. + +While continuing my efforts with profiling and tracing using Tracy, I also +learned the importance of being aware of the memory allocation behavior of the +tracing code (and third-party framework code!) that you integrate into your +application. Tracy has its own allocator, which is great, but one needs to be +mindful of the tracing features enabled and the volume of data being collected +to avoid affecting the real-time behavior of the application being traced. + +## Conclusion + +If you are working on a real-time application -- whether it is an audio +processing application, a low-latency proprietary trading application, or +anything else that needs to have real-time or low-latency behavior, profile your +application in various ways and be aware of the memory usage patterns of your +application. And don't forget to examine the behavior under the hood of ALL +third-party libraries in your application (including tracing or profiling code +or libraries). 🤓 + + + +{% include newsletter.html %} + +{% include submit-pr.html %} + + + +{:.no_toc} + +## Links + +- [GitHub \| Tracy Profiler](https://github.com/wolfpld/tracy) +- [Tracy Profiler Manual (PDF warning)](https://github.com/wolfpld/tracy/releases/latest/download/tracy.pdf) +- [Professional Website \| Bert Schiettecatte](http://www.bertschiettecatte.com/) +- [Noisetron LLC](http://www.noisetron.com/) +- [Percussa](http://www.percussa.com/) diff --git a/img/author/bertschiettecatte.png b/img/author/bertschiettecatte.png new file mode 100644 index 00000000..98c0f22d Binary files /dev/null and b/img/author/bertschiettecatte.png differ diff --git a/img/memory-usage-in-audio-system/percussa.png b/img/memory-usage-in-audio-system/percussa.png new file mode 100644 index 00000000..f2aaa0a7 Binary files /dev/null and b/img/memory-usage-in-audio-system/percussa.png differ diff --git a/img/memory-usage-in-audio-system/trace1.png b/img/memory-usage-in-audio-system/trace1.png new file mode 100644 index 00000000..ad871ac3 Binary files /dev/null and b/img/memory-usage-in-audio-system/trace1.png differ diff --git a/img/memory-usage-in-audio-system/trace2.png b/img/memory-usage-in-audio-system/trace2.png new file mode 100644 index 00000000..dc794e66 Binary files /dev/null and b/img/memory-usage-in-audio-system/trace2.png differ