Merge pull request #84 from rive-app/UR-183-rive-audio

UR-183 Rive Audio

Source/Rive/Private/Game/RiveActor.cpp

@@ -30,6 +30,8 @@ ARiveActor::ARiveActor()
 
 #endif // WITH_EDITOR
 
+	AudioEngine = CreateDefaultSubobject<URiveAudioEngine>(TEXT("RiveAudioEngine"));
+
 	RootComponent->SetMobility(EComponentMobility::Static);
 
 	PrimaryActorTick.bCanEverTick = true;
@@ -68,6 +70,11 @@ void ARiveActor::PostLoad()
 	bEditorDisplayRequested = true;
 
 #endif //WITH_EDITOR
+
+	if (RiveFile)
+	{
+		RiveFile->SetAudioEngine(AudioEngine);
+	}
 }
 
 void ARiveActor::PostActorCreated()
@@ -100,6 +107,7 @@ void ARiveActor::BeginPlay()
 	if (IsValid(RiveFile) && IsValid(ActorWorld) && (ActorWorld->WorldType == EWorldType::PIE))
 	{
 		RiveFile->InstantiateArtboard();
+		RiveFile->GetArtboard()->SetAudioEngine(AudioEngine);
 	}
 
 	Super::BeginPlay();

Source/Rive/Private/Game/RiveActorComponent.cpp

@@ -121,6 +121,11 @@ URiveArtboard* URiveActorComponent::InstantiateArtboard(URiveFile* InRiveFile, c
     URiveArtboard* Artboard = NewObject<URiveArtboard>();
     Artboard->Initialize(InRiveFile->GetNativeFile(), RiveRenderTarget, InArtboardName, InStateMachineName);    
     RenderObjects.Add(Artboard);
+
+    if (RiveAudioEngine != nullptr)
+    {
+        Artboard->SetAudioEngine(RiveAudioEngine);
+    }
 
     return Artboard;
 }

Source/Rive/Private/Rive/RiveFile.cpp

@@ -526,6 +526,32 @@ void URiveFile::InstantiateArtboard(bool bRaiseArtboardChangedEvent)
 		Artboard->Initialize(GetNativeFile(), RiveRenderTarget, ArtboardName, StateMachineName);
 	}
 
+	if (AudioEngine != nullptr)
+	{
+		if (AudioEngine->GetNativeAudioEngine() == nullptr)
+		{
+			if (AudioEngineLambdaHandle.IsValid())
+			{
+				AudioEngine->OnRiveAudioReady.Remove(AudioEngineLambdaHandle);
+				AudioEngineLambdaHandle.Reset();
+			}
+
+			TFunction<void()> AudioLambda = [this]()
+			{
+				Artboard->SetAudioEngine(AudioEngine);
+				AudioEngine->OnRiveAudioReady.Remove(AudioEngineLambdaHandle);
+			};
+			AudioEngineLambdaHandle = AudioEngine->OnRiveAudioReady.AddLambda(AudioLambda);
+		} else
+		{
+			Artboard->SetAudioEngine(AudioEngine);
+		}
+	} else
+	{
+		UE_LOG(LogRive, Warning, TEXT("RiveFile::InstantiateArtboard - AudioEngine is null"));
+	}
+
+
 	Artboard->OnArtboardTick_Render.BindDynamic(this, &URiveFile::OnArtboardTickRender);
 	Artboard->OnGetLocalCoordinate.BindDynamic(this, &URiveFile::GetLocalCoordinate);
 

Source/Rive/Public/Game/RiveActor.h

@@ -5,6 +5,7 @@
 #include "GameFramework/Actor.h"
 #include "RiveActor.generated.h"
 
+class URiveAudioEngine;
 class URiveFile;
 class URiveFullScreenUserWidget;
 
@@ -80,6 +81,9 @@ class RIVE_API ARiveActor : public AActor
 	UPROPERTY(VisibleAnywhere, Instanced, NoClear, Category = "User Interface", meta = (ShowOnlyInnerProperties))
 	TObjectPtr<URiveFullScreenUserWidget> ScreenUserWidget;
 
+	UPROPERTY(VisibleAnywhere, Instanced, NoClear, Category = "Audio", meta = (ShowOnlyInnerProperties))
+	TObjectPtr<URiveAudioEngine> AudioEngine;
+
 #if WITH_EDITORONLY_DATA
 
 	/** Display requested and will be executed on the first frame because we can't call BP function in the loading phase */

Source/Rive/Public/Game/RiveActorComponent.h

@@ -7,11 +7,12 @@
 #include "Components/ActorComponent.h"
 #include "RiveActorComponent.generated.h"
 
+class URiveAudioEngine;
 class URiveTexture;
 class URiveArtboard;
 class URiveFile;
 
-UCLASS(ClassGroup = (Custom), Meta = (BlueprintSpawnableComponent))
+UCLASS(ClassGroup = (Rive), Meta = (BlueprintSpawnableComponent))
 class RIVE_API URiveActorComponent : public UActorComponent
 {
     DECLARE_DYNAMIC_MULTICAST_DELEGATE(FRiveReadyDelegate);
@@ -69,6 +70,9 @@ class RIVE_API URiveActorComponent : public UActorComponent
     UPROPERTY(BlueprintReadWrite, Transient, Category = Rive)
     TObjectPtr<URiveTexture> RenderTarget;
 
+    UPROPERTY(BlueprintReadWrite, Transient, Category = Rive)
+    TObjectPtr<URiveAudioEngine> RiveAudioEngine;
+
 private:
     UE::Rive::Renderer::IRiveRenderTargetPtr RiveRenderTarget;
 };

Source/Rive/Public/Rive/RiveFile.h

@@ -196,6 +196,8 @@ class RIVE_API URiveFile : public URiveTexture, public FTickableGameObject
 		return nullptr;
 	}
 
+	void SetAudioEngine(URiveAudioEngine* InAudioEngine) { AudioEngine = InAudioEngine; }
+
 	UPROPERTY(VisibleAnywhere, Category=Rive)
 	TObjectPtr<URiveFile> ParentRiveFile;
 
@@ -259,7 +261,11 @@ class RIVE_API URiveFile : public URiveTexture, public FTickableGameObject
 
 	UPROPERTY(Transient, VisibleInstanceOnly, BlueprintReadOnly, Category=Rive, meta=(NoResetToDefault, AllowPrivateAccess, ShowInnerProperties))
 	URiveArtboard* Artboard = nullptr;
-
+
+	UPROPERTY(Transient, VisibleInstanceOnly, BlueprintReadOnly, Category=Rive, meta=(AllowPrivateAccess))
+	URiveAudioEngine* AudioEngine = nullptr;
+	FDelegateHandle AudioEngineLambdaHandle;
+
 	rive::Span<const uint8> RiveNativeFileSpan;
 
 	rive::Span<const uint8>& GetNativeFileSpan()

Source/RiveCore/Private/Assets/RiveAsset.cpp

@@ -4,12 +4,28 @@
 #include "Logs/RiveCoreLog.h"
 #include "Misc/FileHelper.h"
 #include "rive/factory.hpp"
+#include "rive/assets/audio_asset.hpp"
+#include "rive/audio/audio_source.hpp"
 
 void URiveAsset::PostLoad()
 {
 	UObject::PostLoad();
 
-	// NativeAsset.Bytes = &NativeAssetBytes;
+	// Older version of UE Rive used these values as enum values
+	// Newer version of UE Rive doesn't use these values as enum values because the type values are beyond uint8 space
+	// This ensures older rive assets will still work, by converting old values to new values
+	switch((int)Type)
+	{
+	case 103:
+		Type = ERiveAssetType::FileBase;
+		break;
+	case 105:
+		Type = ERiveAssetType::Image;
+		break;
+	case 141:
+		Type = ERiveAssetType::Font;
+		break;
+	}
 }
 
 void URiveAsset::LoadFromDisk()
@@ -20,14 +36,16 @@ void URiveAsset::LoadFromDisk()
 	}
 }
 
-bool URiveAsset::DecodeNativeAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes)
+bool URiveAsset::LoadNativeAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes)
 {
 	switch(Type)
 	{
 	case ERiveAssetType::Font:
 		return DecodeFontAsset(InAsset, InRiveFactory, AssetBytes);
 	case ERiveAssetType::Image:
 		return DecodeImageAsset(InAsset, InRiveFactory, AssetBytes);
+	case ERiveAssetType::Audio:
+		return LoadAudioAsset(InAsset, InRiveFactory, AssetBytes);
 	}
 
 	return false;
@@ -65,3 +83,15 @@ bool URiveAsset::DecodeFontAsset(rive::FileAsset& InAsset, rive::Factory* InRive
 	NativeAsset = FontAsset;
 	return true;
 }
+
+bool URiveAsset::LoadAudioAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes)
+{
+	rive::SimpleArray<uint8_t> Data = rive::SimpleArray(AssetBytes.data(), AssetBytes.count());
+	rive::AudioSource* AudioSource = new rive::AudioSource(Data);
+	rive::rcp<rive::AudioSource> RcpAudioSource = rive::rcp(AudioSource);
+
+	rive::AudioAsset* AudioAsset = InAsset.as<rive::AudioAsset>();
+	AudioAsset->audioSource(RcpAudioSource);
+	NativeAsset = AudioAsset;
+	return true;
+}

Source/RiveCore/Private/Assets/URAssetHelpers.cpp

@@ -44,6 +44,9 @@ bool URAssetHelpers::FindDiskAsset(const FString& InBasePath, URiveAsset* InRive
 	case ERiveAssetType::Image:
 		Extensions = &ImageExtensions;
 		break;
+	case ERiveAssetType::Audio:
+		Extensions = &AudioExtensions;
+		break;
 	default:
 		assert(true);
 		return false;  // this means we don't support this asset type
@@ -77,4 +80,21 @@ bool URAssetHelpers::FindDiskAsset(const FString& InBasePath, URiveAsset* InRive
 
 	InRiveAsset->AssetPath = FilePath;
 	return true;
-}
+}
+
+ERiveAssetType URAssetHelpers::GetUnrealType(uint16_t RiveType)
+{
+	switch(RiveType)
+	{
+	case 103:
+		return ERiveAssetType::FileBase;
+	case 105:
+		return ERiveAssetType::Image;
+	case 141:
+		return ERiveAssetType::Font;
+	case 406:
+		return ERiveAssetType::Audio;
+	default:
+		return ERiveAssetType::None;
+	}
+}

Source/RiveCore/Private/Assets/URAssetImporter.cpp

@@ -93,7 +93,7 @@ bool UE::Rive::Assets::FURAssetImporter::loadContents(rive::FileAsset& InAsset,
 
 	RiveAsset->Id = InAsset.assetId();
 	RiveAsset->Name = FString(UTF8_TO_TCHAR(InAsset.name().c_str()));
-	RiveAsset->Type = static_cast<ERiveAssetType>(InAsset.coreType());
+	RiveAsset->Type = URAssetHelpers::GetUnrealType(InAsset.coreType()); 
 	RiveAsset->bIsInBand = bIsInBand;
 	RiveAsset->MarkPackageDirty();
 

Source/RiveCore/Private/Assets/URFileAssetLoader.cpp

@@ -2,6 +2,7 @@
 
 #include "Assets/URFileAssetLoader.h"
 #include "Assets/RiveAsset.h"
+#include "Assets/URAssetHelpers.h"
 #include "Logs/RiveCoreLog.h"
 #include "rive/factory.hpp"
 
@@ -40,7 +41,11 @@ bool UE::Rive::Assets::FURFileAssetLoader::loadContents(rive::FileAsset& InAsset
 
 	URiveAsset* RiveAsset = nullptr;
 	const TObjectPtr<URiveAsset>* RiveAssetPtr = Assets.Find(InAsset.assetId());
-	if (RiveAssetPtr == nullptr)
+	if (RiveAssetPtr != nullptr && RiveAssetPtr->Get() != nullptr)
+	{
+		RiveAsset = RiveAssetPtr->Get();
+	}
+	else
 	{
 		if (!bUseInBand)
 		{
@@ -54,12 +59,11 @@ bool UE::Rive::Assets::FURFileAssetLoader::loadContents(rive::FileAsset& InAsset
 
 		RiveAsset->Id = InAsset.assetId();
 		RiveAsset->Name = FString(UTF8_TO_TCHAR(InAsset.name().c_str()));
-		RiveAsset->Type = static_cast<ERiveAssetType>(InAsset.coreType());
+		RiveAsset->Type = URAssetHelpers::GetUnrealType(InAsset.coreType()); 
 		RiveAsset->bIsInBand = true;
-	}
-	else
-	{
-		RiveAsset = RiveAssetPtr->Get();
+
+		// We only add it to our assets here so that it shows up in the inspector, otherwise this doesn't have any functional effect on anything due to it being a transient, in-band asset
+		Assets.Add(InAsset.assetId(), RiveAsset);
 	}
 
 	rive::Span<const uint8> OutOfBandBytes;
@@ -74,7 +78,7 @@ bool UE::Rive::Assets::FURFileAssetLoader::loadContents(rive::FileAsset& InAsset
 		AssetBytes = &OutOfBandBytes;
 	}
 
-	return RiveAsset->DecodeNativeAsset(InAsset, InFactory, *AssetBytes);
+	return RiveAsset->LoadNativeAsset(InAsset, InFactory, *AssetBytes);
 }
 
 #endif // WITH_RIVE

Source/RiveCore/Private/RiveArtboard.cpp

@@ -401,6 +401,22 @@ void URiveArtboard::Initialize(rive::File* InNativeFilePtr, UE::Rive::Renderer::
 	Initialize_Internal(NativeArtboard);
 }
 
+void URiveArtboard::SetAudioEngine(URiveAudioEngine* AudioEngine)
+{
+	if (AudioEngine == nullptr)
+	{
+		rive::rcp<rive::AudioEngine> NativeEngine = NativeArtboardPtr->audioEngine();
+
+		if (NativeEngine != nullptr)
+		{
+			NativeEngine->unref();
+		}
+		NativeArtboardPtr->audioEngine(nullptr);
+		return;
+	}
+	NativeArtboardPtr->audioEngine(AudioEngine->GetNativeAudioEngine());
+}
+
 void URiveArtboard::Tick_Render(float InDeltaSeconds)
 {
 	if (OnArtboardTick_Render.IsBound())

Source/RiveCore/Private/RiveAudioEngine.cpp

@@ -0,0 +1,51 @@
+// Copyright Rive, Inc. All rights reserved.
+
+#include "RiveAudioEngine.h"
+#include "AudioDevice.h"
+#include "rive/artboard.hpp"
+
+void URiveAudioEngine::BeginPlay()
+{
+	if (FAudioDevice* AudioDevice = GetAudioDevice())
+	{
+		// Make our Rive audio engine
+		if (NativeAudioEnginePtr.get() != nullptr)
+		{
+			NativeAudioEnginePtr->unref();
+			NativeAudioEnginePtr = nullptr;
+		}
+		NumChannels = 2;
+
+		NativeAudioEnginePtr = rive::rcp(rive::AudioEngine::Make(NumChannels, AudioDevice->SampleRate));
+		Start();
+	}
+
+	OnRiveAudioReady.Broadcast();
+	Super::BeginPlay();
+}
+
+void URiveAudioEngine::BeginDestroy()
+{
+	if (NativeAudioEnginePtr != nullptr) {
+		NativeAudioEnginePtr->unref();
+		NativeAudioEnginePtr = nullptr;
+	}
+
+	Super::BeginDestroy();
+
+}
+
+int32 URiveAudioEngine::OnGenerateAudio(float* OutAudio, int32 NumSamples)
+{
+	if (NativeAudioEnginePtr == nullptr) return 0;
+
+	TArray<float> AudioData;
+	AudioData.AddZeroed(NumSamples);
+
+	if (NativeAudioEnginePtr->readAudioFrames(AudioData.GetData(), NumSamples / NumChannels, nullptr))
+	{
+		FMemory::Memcpy(OutAudio, AudioData.GetData(), NumSamples * sizeof(float));
+		return NumSamples;
+	}
+	return 0;
+}

Source/RiveCore/Public/Assets/RiveAsset.h

@@ -18,9 +18,10 @@ UENUM(BlueprintType)
 enum class ERiveAssetType : uint8
 {
 	None = 0,
-	FileBase = 103,
-	Image = 105,
-	Font = 141
+	FileBase,
+	Image,
+	Font,
+	Audio
 };
 
 /**
@@ -62,8 +63,9 @@ class RIVECORE_API URiveAsset : public UObject
 #endif
 
 	void LoadFromDisk();
-	bool DecodeNativeAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes);
+	bool LoadNativeAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes);
 private:
 	bool DecodeImageAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes);
 	bool DecodeFontAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes);
+	bool LoadAudioAsset(rive::FileAsset& InAsset, rive::Factory* InRiveFactory, const rive::Span<const uint8>& AssetBytes);
 };