From 63351e42fae40f94f191191eb949707b95073bca Mon Sep 17 00:00:00 2001
From: jjangga0214 <bnbcmindnpass@gmail.com>
Date: Wed, 9 Oct 2024 15:11:28 +0900
Subject: [PATCH 1/2] feat(openai): introducing `KnownRealtimeResponse`

---
 plugins/openai/src/realtime/api_proto.ts      | 44 +++++++++++--------
 plugins/openai/src/realtime/realtime_model.ts | 27 +++++++++++-
 plugins/openai/src/utility-types.ts           |  1 +
 3 files changed, 53 insertions(+), 19 deletions(-)
 create mode 100644 plugins/openai/src/utility-types.ts

diff --git a/plugins/openai/src/realtime/api_proto.ts b/plugins/openai/src/realtime/api_proto.ts
index 382582f6..c7555cb3 100644
--- a/plugins/openai/src/realtime/api_proto.ts
+++ b/plugins/openai/src/realtime/api_proto.ts
@@ -2,6 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+import { ExtractStrict } from "../utility-types";
+
 export const SAMPLE_RATE = 24000;
 export const NUM_CHANNELS = 1;
 export const IN_FRAME_SIZE = 2400; // 100ms
@@ -18,13 +20,13 @@ export type InputTranscriptionModel = 'whisper-1' | string; // Open-ended, for f
 export type Modality = 'text' | 'audio';
 export type ToolChoice = 'auto' | 'none' | 'required' | string;
 export type State = 'initializing' | 'listening' | 'thinking' | 'speaking' | string;
-export type ResponseStatus =
+export type KnownResponseStatus =
   | 'in_progress'
   | 'completed'
   | 'incomplete'
   | 'cancelled'
-  | 'failed'
-  | string;
+  | 'failed';
+export type ResponseStatus = KnownResponseStatus | string;
 export type ClientEventType =
   | 'session.update'
   | 'input_audio_buffer.append'
@@ -191,22 +193,28 @@ export interface ConversationResource {
   object: 'realtime.conversation';
 }
 
+export interface IncompleteResponseStatusDetails {
+  type: ExtractStrict<KnownResponseStatus, 'incomplete'>;
+  reason: 'max_output_tokens' | 'content_filter' | string;
+}
+
+export interface FailedResponseStatusDetails {
+  type: ExtractStrict<KnownResponseStatus, 'failed'>;
+  error?: {
+    code: 'server_error' | 'rate_limit_exceeded' | string;
+    message: string;
+  };
+}
+
+export interface CancelledResponseStatusDetails {
+  type: ExtractStrict<KnownResponseStatus, 'cancelled'>;
+  reason: 'turn_detected' | 'client_cancelled' | string;
+}
+
 export type ResponseStatusDetails =
-  | {
-      type: 'incomplete';
-      reason: 'max_output_tokens' | 'content_filter' | string;
-    }
-  | {
-      type: 'failed';
-      error?: {
-        code: 'server_error' | 'rate_limit_exceeded' | string;
-        message: string;
-      };
-    }
-  | {
-      type: 'cancelled';
-      reason: 'turn_detected' | 'client_cancelled' | string;
-    };
+  | IncompleteResponseStatusDetails
+  | FailedResponseStatusDetails
+  | CancelledResponseStatusDetails;
 
 export interface ResponseResource {
   id: string;
diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts
index dfd14ccb..36c8a963 100644
--- a/plugins/openai/src/realtime/realtime_model.ts
+++ b/plugins/openai/src/realtime/realtime_model.ts
@@ -6,6 +6,7 @@ import { llm, log, multimodal } from '@livekit/agents';
 import { AudioFrame } from '@livekit/rtc-node';
 import { once } from 'events';
 import { WebSocket } from 'ws';
+import { ExtractStrict } from '../utility-types.js';
 import * as api_proto from './api_proto.js';
 
 interface ModelOptions {
@@ -23,7 +24,7 @@ interface ModelOptions {
   baseURL: string;
 }
 
-export interface RealtimeResponse {
+interface RealtimeResponseBase {
   id: string;
   status: api_proto.ResponseStatus;
   statusDetails: api_proto.ResponseStatusDetails | null;
@@ -31,6 +32,30 @@ export interface RealtimeResponse {
   doneFut: Future;
 }
 
+interface KnownRealtimeResponseBase<T extends api_proto.ResponseStatusDetails>
+  extends RealtimeResponseBase {
+  status: T['type'];
+  statusDetails: T;
+}
+
+export interface UnknownRealtimeResponse extends RealtimeResponseBase {
+  status: string;
+  statusDetails: null;
+}
+
+export type IncompleteRealtimeResponse =
+  KnownRealtimeResponseBase<api_proto.IncompleteResponseStatusDetails>;
+export type FailedRealtimeResponse =
+  KnownRealtimeResponseBase<api_proto.FailedResponseStatusDetails>;
+export type CancelledRealtimeResponse =
+  KnownRealtimeResponseBase<api_proto.CancelledResponseStatusDetails>;
+export type KnownRealtimeResponse =
+  | IncompleteRealtimeResponse
+  | FailedRealtimeResponse
+  | CancelledRealtimeResponse;
+
+export type RealtimeResponse = KnownRealtimeResponse | UnknownRealtimeResponse;
+
 export interface RealtimeOutput {
   responseId: string;
   itemId: string;
diff --git a/plugins/openai/src/utility-types.ts b/plugins/openai/src/utility-types.ts
new file mode 100644
index 00000000..3002f8e1
--- /dev/null
+++ b/plugins/openai/src/utility-types.ts
@@ -0,0 +1 @@
+export type ExtractStrict<T, U extends T> = T extends U ? T : never;

From d0e04d153cf082ee7b5aefa878e813031488d8b9 Mon Sep 17 00:00:00 2001
From: jjangga0214 <bnbcmindnpass@gmail.com>
Date: Wed, 9 Oct 2024 17:01:03 +0900
Subject: [PATCH 2/2] refactor(openai): rm unused import

---
 plugins/openai/src/realtime/realtime_model.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts
index 36c8a963..d8bb9841 100644
--- a/plugins/openai/src/realtime/realtime_model.ts
+++ b/plugins/openai/src/realtime/realtime_model.ts
@@ -6,7 +6,6 @@ import { llm, log, multimodal } from '@livekit/agents';
 import { AudioFrame } from '@livekit/rtc-node';
 import { once } from 'events';
 import { WebSocket } from 'ws';
-import { ExtractStrict } from '../utility-types.js';
 import * as api_proto from './api_proto.js';
 
 interface ModelOptions {