1import type {
2 AttachmentsConfig,
3 RenderToolCallConfig,
4} from "@copilotkit/angular";
5import {
6 ChangeDetectionStrategy,
7 Component,
8 effect,
9 inject,
10 signal,
11 viewChild,
12} from "@angular/core";
13import { ActivatedRoute } from "@angular/router";
14import {
15 CopilotKit,
16 injectAgentStore,
17 registerRenderToolCall,
18} from "@copilotkit/angular";
19import { z } from "zod";
20
21import { agentIdForCurrentIntegration } from "../../feature-agent";
22import { FeatureHeaderComponent } from "../feature-header.component";
23import { ShowcaseChatHostComponent } from "../showcase-chat-host.component";
24import { WeatherToolCard } from "../tools/tool-cards";
25import {
26 createMultimodalMessage,
27 dedupeUserMessageMedia,
28 validateSampleBytes,
29 VOICE_WEATHER_TOOL_NAMES,
30} from "./media-model";
31import type { MediaAgentMessage, SampleSpec } from "./media-model";
32
33const VOICE_SAMPLE_TEXT = "What is the weather in Tokyo?";
34
35type VoiceWeatherArgs = Record<string, unknown>;
36
37
38export const voiceWeatherRendererConfigs: readonly RenderToolCallConfig<VoiceWeatherArgs>[] =
39 VOICE_WEATHER_TOOL_NAMES.map((name) => ({
40 name,
41 args: z.record(z.unknown()),
42 component:
43 WeatherToolCard as unknown as RenderToolCallConfig<VoiceWeatherArgs>["component"],
44 }));
45
46const SAMPLES: readonly SampleSpec[] = [
47 {
48 buttonLabel: "Try with sample image",
49 filename: "sample.png",
50 mimeType: "image/png",
51 testId: "multimodal-sample-image-button",
52 fetchUrl: "/demo-files/sample.png",
53 autoPrompt: "can you tell me what is in this demo image I just attached",
54 },
55 {
56 buttonLabel: "Try with sample PDF",
57 filename: "sample.pdf",
58 mimeType: "application/pdf",
59 testId: "multimodal-sample-pdf-button",
60 fetchUrl: "/demo-files/sample.pdf",
61 autoPrompt: "can you tell me what is in this demo pdf I just attached",
62 },
63];
64
65const MULTIMODAL_ATTACHMENTS: AttachmentsConfig = {
66 enabled: true,
67 accept: "image/*,application/pdf",
68 maxSize: 10 * 1024 * 1024,
69};
70
71@Component({
72 selector: "showcase-media-feature",
73 imports: [FeatureHeaderComponent, ShowcaseChatHostComponent],
74 changeDetection: ChangeDetectionStrategy.OnPush,
75 host: { class: "feature-page" },
76 template: `
77 <showcase-feature-header />
78 <main class="media-page">
79 <section class="sample-panel" aria-label="Bundled media samples">
80 @if (feature === "voice") {
81 <div class="sample-copy">
82 <strong>Voice input</strong>
83 <span>Use the microphone, or insert a deterministic sample.</span>
84 </div>
85 <button
86 type="button"
87 data-testid="voice-sample-audio-button"
88 [title]="'Inserts: "' + voiceSampleText + '"'"
89 (click)="insertVoiceSample()"
90 >
91 <span aria-hidden="true">🎙</span>
92 <span>Try a sample audio</span>
93 </button>
94 } @else {
95 <div class="sample-copy">
96 <strong>Bundled samples</strong>
97 <span>Send a real image or PDF through the agent runtime.</span>
98 </div>
99 <div class="sample-actions">
100 @for (sample of samples; track sample.testId) {
101 <button
102 type="button"
103 [attr.data-testid]="sample.testId"
104 [disabled]="loading() !== null"
105 (click)="sendSample(sample)"
106 >
107 {{ loading() === sample.testId ? "Sending…" : sample.buttonLabel }}
108 </button>
109 }
110 </div>
111 }
112 @if (error()) {
113 <p class="sample-error" role="alert">{{ error() }}</p>
114 }
115 </section>
116 <section class="chat-surface" aria-label="CopilotKit assistant">
117 <showcase-chat-host
118 [agentId]="agentId"
119 [attachments]="
120 feature === 'multimodal' ? multimodalAttachments : undefined
121 "
122 />
123 </section>
124 </main>
125 `,
126 styles: `
127 .media-page {
128 display: grid;
129 min-height: 0;
130 grid-template-rows: auto minmax(0, 1fr);
131 gap: 0.75rem;
132 padding: 0.75rem;
133 background: light-dark(#eef3f7, #0a0a0a);
134 }
135 .sample-panel {
136 display: flex;
137 flex-wrap: wrap;
138 align-items: center;
139 gap: 0.75rem 1rem;
140 padding: 0.75rem 1rem;
141 border: 1px solid light-dark(#d8e0ea, #303136);
142 border-radius: 0.9rem;
143 background: light-dark(#fff, #191a1e);
144 }
145 .sample-copy {
146 display: grid;
147 flex: 1 1 16rem;
148 gap: 0.15rem;
149 color: light-dark(#14213d, #fafafa);
150 }
151 .sample-copy span {
152 color: light-dark(#52637a, #adadb2);
153 font-size: 0.82rem;
154 }
155 .sample-actions {
156 display: flex;
157 flex-wrap: wrap;
158 gap: 0.5rem;
159 }
160 button {
161 display: inline-flex;
162 align-items: center;
163 gap: 0.4rem;
164 padding: 0.55rem 0.8rem;
165 border: 1px solid light-dark(#c7d2e0, #303136);
166 border-radius: 0.65rem;
167 color: light-dark(#20324d, #fafafa);
168 background: light-dark(#fff, #191a1e);
169 font: inherit;
170 font-size: 0.84rem;
171 font-weight: 650;
172 cursor: pointer;
173 }
174 button:hover:not(:disabled) {
175 border-color: #4263eb;
176 background: light-dark(#f5f7ff, rgb(30 27 75 / 40%));
177 }
178 button:focus-visible {
179 outline: 3px solid #91a7ff;
180 outline-offset: 2px;
181 }
182 button:disabled {
183 cursor: wait;
184 opacity: 0.62;
185 }
186 .sample-error {
187 flex-basis: 100%;
188 margin: 0;
189 color: light-dark(#991b1b, #fca5a5);
190 font-size: 0.84rem;
191 }
192 .chat-surface {
193 min-height: 0;
194 overflow: hidden;
195 border: 1px solid light-dark(#d8e0ea, #303136);
196 border-radius: 1rem;
197 background: light-dark(#fff, #191a1e);
198 }
199 `,
200})
201export class MediaFeatureComponent {
202 private readonly route = inject(ActivatedRoute);
203 private readonly copilotKit = inject(CopilotKit);
204 private readonly chatHost = viewChild.required(ShowcaseChatHostComponent);
205 protected readonly feature =
206 (this.route.snapshot.data["feature"] as string | undefined) ?? "voice";
207 protected readonly agentId = agentIdForCurrentIntegration(this.feature);
208 private readonly agentStore = injectAgentStore(this.agentId);
209 protected readonly voiceSampleText = VOICE_SAMPLE_TEXT;
210 protected readonly samples = SAMPLES;
211 protected readonly multimodalAttachments = MULTIMODAL_ATTACHMENTS;
212 protected readonly loading = signal<string | null>(null);
213 protected readonly error = signal<string | null>(null);
214
215 constructor() {
216 if (this.feature === "voice") {
217 for (const config of voiceWeatherRendererConfigs) {
218 registerRenderToolCall(config);
219 }
220 }
221 if (this.feature === "multimodal") {
222 effect((onCleanup) => {
223 const handle = installMediaDedupe(this.agentStore().agent);
224 onCleanup(() => handle.unsubscribe());
225 });
226 }
227 }
228
229
230 protected insertVoiceSample(): void {
231 if (!this.chatHost().populateComposer(VOICE_SAMPLE_TEXT)) {
232 this.error.set("The chat composer is not ready yet. Try again.");
233 return;
234 }
235 this.error.set(null);
236 }
237
238
239 protected async sendSample(spec: SampleSpec): Promise<void> {
240 if (this.loading() !== null) return;
241 this.loading.set(spec.testId);
242 this.error.set(null);
243 try {
244 const response = await fetch(spec.fetchUrl);
245 if (!response.ok) {
246 throw new Error(
247 `Could not fetch sample "${spec.filename}" (HTTP ${response.status}).`,
248 );
249 }
250 const buffer = await response.arrayBuffer();
251 const bytes = new Uint8Array(buffer);
252 validateSampleBytes(bytes, spec.mimeType, spec.filename);
253 const base64 = await bufferToBase64(buffer, spec.mimeType);
254 const agent = this.agentStore().agent;
255 const message = createMultimodalMessage(
256 spec,
257 base64,
258 buffer.byteLength,
259 createMessageId(),
260 );
261 agent.addMessage(message as Parameters<typeof agent.addMessage>[0]);
262 await this.copilotKit.core.runAgent({ agent });
263 } catch (error: unknown) {
264 console.error("[showcase-angular:multimodal] Sample send failed", error);
265 this.error.set(
266 error instanceof Error ? error.message : "The sample send failed.",
267 );
268 } finally {
269 this.loading.set(null);
270 }
271 }
272}
273
274interface SubscribableAgent {
275 subscribe: (subscriber: unknown) => { unsubscribe: () => void };
276}
277
278284
285function installMediaDedupe(agent: object): {
286 unsubscribe: () => void;
287} {
288 const dedupe = ({
289 messages,
290 }: {
291 messages: ReadonlyArray<Readonly<MediaAgentMessage>>;
292 }) => {
293 const deduped = dedupeUserMessageMedia(messages);
294 return deduped ? { messages: deduped } : undefined;
295 };
296 return (agent as SubscribableAgent).subscribe({
297 onMessagesSnapshotEvent: dedupe,
298 onRunFinalized: dedupe,
299 });
300}
301
302
303function bufferToBase64(
304 buffer: ArrayBuffer,
305 mimeType: string,
306): Promise<string> {
307 return new Promise((resolve, reject) => {
308 const reader = new FileReader();
309 reader.addEventListener(
310 "error",
311 () =>
312 reject(
313 reader.error ?? new Error("The selected media could not be read."),
314 ),
315 { once: true },
316 );
317 reader.addEventListener(
318 "load",
319 () => {
320 if (typeof reader.result !== "string") {
321 reject(new Error("The selected media returned an invalid result."));
322 return;
323 }
324 const comma = reader.result.indexOf(",");
325 resolve(comma >= 0 ? reader.result.slice(comma + 1) : reader.result);
326 },
327 { once: true },
328 );
329 reader.readAsDataURL(new Blob([buffer], { type: mimeType }));
330 });
331}
332
333
334function createMessageId(): string {
335 return (
336 globalThis.crypto?.randomUUID?.() ??
337 `angular-${Date.now()}-${Math.random().toString(16).slice(2)}`
338 );
339}
340