1import type {
2 AttachmentsConfig,
3 RenderToolCallConfig,
4} from "@copilotkit/angular";
5import {
6 ChangeDetectionStrategy,
7 Component,
8 effect,
9 inject,
10 signal,
11 viewChild,
12} from "@angular/core";
13import { ActivatedRoute } from "@angular/router";
14import {
15 CopilotKit,
16 injectAgentStore,
17 registerRenderToolCall,
18} from "@copilotkit/angular";
19import { z } from "zod";
20
21import { agentIdForRoute } from "../../feature-agent";
22import { FeatureHeaderComponent } from "../feature-header.component";
23import { ShowcaseChatHostComponent } from "../showcase-chat-host.component";
24import { WeatherToolCard } from "../tools/tool-cards";
25import {
26 createMultimodalMessage,
27 dedupeUserMessageMedia,
28 rewriteMessagesForLegacyConverter,
29 validateSampleBytes,
30 VOICE_WEATHER_TOOL_NAMES,
31} from "./media-model";
32import type { MediaAgentMessage, SampleSpec } from "./media-model";
33
34const VOICE_SAMPLE_TEXT = "What is the weather in Tokyo?";
35
36type VoiceWeatherArgs = Record<string, unknown>;
37
38
39export const voiceWeatherRendererConfigs: readonly RenderToolCallConfig<VoiceWeatherArgs>[] =
40 VOICE_WEATHER_TOOL_NAMES.map((name) => ({
41 name,
42 args: z.record(z.unknown()),
43 component:
44 WeatherToolCard as unknown as RenderToolCallConfig<VoiceWeatherArgs>["component"],
45 }));
46
47const SAMPLES: readonly SampleSpec[] = [
48 {
49 buttonLabel: "Try with sample image",
50 filename: "sample.png",
51 mimeType: "image/png",
52 testId: "multimodal-sample-image-button",
53 fetchUrl: "/demo-files/sample.png",
54 autoPrompt: "can you tell me what is in this demo image I just attached",
55 },
56 {
57 buttonLabel: "Try with sample PDF",
58 filename: "sample.pdf",
59 mimeType: "application/pdf",
60 testId: "multimodal-sample-pdf-button",
61 fetchUrl: "/demo-files/sample.pdf",
62 autoPrompt: "can you tell me what is in this demo pdf I just attached",
63 },
64];
65
66const MULTIMODAL_ATTACHMENTS: AttachmentsConfig = {
67 enabled: true,
68 accept: "image/*,application/pdf",
69 maxSize: 10 * 1024 * 1024,
70};
71
72@Component({
73 selector: "showcase-media-feature",
74 imports: [FeatureHeaderComponent, ShowcaseChatHostComponent],
75 changeDetection: ChangeDetectionStrategy.OnPush,
76 host: { class: "feature-page" },
77 template: `
78 <showcase-feature-header />
79 <main class="media-page">
80 <section class="sample-panel" aria-label="Bundled media samples">
81 @if (feature === "voice") {
82 <div class="sample-copy">
83 <strong>Voice input</strong>
84 <span>Use the microphone, or insert a deterministic sample.</span>
85 </div>
86 <button
87 type="button"
88 data-testid="voice-sample-audio-button"
89 [title]="'Inserts: "' + voiceSampleText + '"'"
90 (click)="insertVoiceSample()"
91 >
92 <span aria-hidden="true">🎙</span>
93 <span>Try a sample audio</span>
94 </button>
95 } @else {
96 <div class="sample-copy">
97 <strong>Bundled samples</strong>
98 <span>Send a real image or PDF through the agent runtime.</span>
99 </div>
100 <div class="sample-actions">
101 @for (sample of samples; track sample.testId) {
102 <button
103 type="button"
104 [attr.data-testid]="sample.testId"
105 [disabled]="loading() !== null"
106 (click)="sendSample(sample)"
107 >
108 {{ loading() === sample.testId ? "Sending…" : sample.buttonLabel }}
109 </button>
110 }
111 </div>
112 }
113 @if (error()) {
114 <p class="sample-error" role="alert">{{ error() }}</p>
115 }
116 </section>
117 <section class="chat-surface" aria-label="CopilotKit assistant">
118 <showcase-chat-host
119 [agentId]="agentId"
120 [attachments]="
121 feature === 'multimodal' ? multimodalAttachments : undefined
122 "
123 />
124 </section>
125 </main>
126 `,
127 styles: `
128 .media-page {
129 display: grid;
130 min-height: 0;
131 grid-template-rows: auto minmax(0, 1fr);
132 gap: 0.75rem;
133 padding: 0.75rem;
134 background: #eef3f7;
135 }
136 .sample-panel {
137 display: flex;
138 flex-wrap: wrap;
139 align-items: center;
140 gap: 0.75rem 1rem;
141 padding: 0.75rem 1rem;
142 border: 1px solid #d8e0ea;
143 border-radius: 0.9rem;
144 background: #fff;
145 }
146 .sample-copy {
147 display: grid;
148 flex: 1 1 16rem;
149 gap: 0.15rem;
150 color: #14213d;
151 }
152 .sample-copy span {
153 color: #52637a;
154 font-size: 0.82rem;
155 }
156 .sample-actions {
157 display: flex;
158 flex-wrap: wrap;
159 gap: 0.5rem;
160 }
161 button {
162 display: inline-flex;
163 align-items: center;
164 gap: 0.4rem;
165 padding: 0.55rem 0.8rem;
166 border: 1px solid #c7d2e0;
167 border-radius: 0.65rem;
168 color: #20324d;
169 background: #fff;
170 font: inherit;
171 font-size: 0.84rem;
172 font-weight: 650;
173 cursor: pointer;
174 }
175 button:hover:not(:disabled) {
176 border-color: #4263eb;
177 background: #f5f7ff;
178 }
179 button:focus-visible {
180 outline: 3px solid #91a7ff;
181 outline-offset: 2px;
182 }
183 button:disabled {
184 cursor: wait;
185 opacity: 0.62;
186 }
187 .sample-error {
188 flex-basis: 100%;
189 margin: 0;
190 color: #991b1b;
191 font-size: 0.84rem;
192 }
193 .chat-surface {
194 min-height: 0;
195 overflow: hidden;
196 border: 1px solid #d8e0ea;
197 border-radius: 1rem;
198 background: #fff;
199 }
200 `,
201})
202export class MediaFeatureComponent {
203 private readonly route = inject(ActivatedRoute);
204 private readonly copilotKit = inject(CopilotKit);
205 private readonly chatHost = viewChild.required(ShowcaseChatHostComponent);
206 protected readonly feature =
207 (this.route.snapshot.data["feature"] as string | undefined) ?? "voice";
208 protected readonly agentId = agentIdForRoute(this.feature, this.route);
209 private readonly agentStore = injectAgentStore(this.agentId);
210 protected readonly voiceSampleText = VOICE_SAMPLE_TEXT;
211 protected readonly samples = SAMPLES;
212 protected readonly multimodalAttachments = MULTIMODAL_ATTACHMENTS;
213 protected readonly loading = signal<string | null>(null);
214 protected readonly error = signal<string | null>(null);
215
216 constructor() {
217 if (this.feature === "voice") {
218 for (const config of voiceWeatherRendererConfigs) {
219 registerRenderToolCall(config);
220 }
221 }
222 if (this.feature === "multimodal") {
223 effect((onCleanup) => {
224 const handle = installLegacyConverterShim(this.agentStore().agent);
225 onCleanup(() => handle.unsubscribe());
226 });
227 }
228 }
229
230
231 protected insertVoiceSample(): void {
232 if (!this.chatHost().populateComposer(VOICE_SAMPLE_TEXT)) {
233 this.error.set("The chat composer is not ready yet. Try again.");
234 return;
235 }
236 this.error.set(null);
237 }
238
239
240 protected async sendSample(spec: SampleSpec): Promise<void> {
241 if (this.loading() !== null) return;
242 this.loading.set(spec.testId);
243 this.error.set(null);
244 try {
245 const response = await fetch(spec.fetchUrl);
246 if (!response.ok) {
247 throw new Error(
248 `Could not fetch sample "${spec.filename}" (HTTP ${response.status}).`,
249 );
250 }
251 const buffer = await response.arrayBuffer();
252 const bytes = new Uint8Array(buffer);
253 validateSampleBytes(bytes, spec.mimeType, spec.filename);
254 const base64 = await bufferToBase64(buffer, spec.mimeType);
255 const agent = this.agentStore().agent;
256 const message = createMultimodalMessage(
257 spec,
258 base64,
259 buffer.byteLength,
260 createMessageId(),
261 );
262 agent.addMessage(message as Parameters<typeof agent.addMessage>[0]);
263 await this.copilotKit.core.runAgent({ agent });
264 } catch (error: unknown) {
265 console.error("[showcase-angular:multimodal] Sample send failed", error);
266 this.error.set(
267 error instanceof Error ? error.message : "The sample send failed.",
268 );
269 } finally {
270 this.loading.set(null);
271 }
272 }
273}
274
275interface SubscribableAgent {
276 subscribe: (subscriber: unknown) => { unsubscribe: () => void };
277}
278
279
280function installLegacyConverterShim(agent: object): {
281 unsubscribe: () => void;
282} {
283 const rewrite = ({
284 messages,
285 }: {
286 messages: ReadonlyArray<Readonly<MediaAgentMessage>>;
287 }) => {
288 const rewritten = rewriteMessagesForLegacyConverter(messages);
289 return rewritten ? { messages: rewritten } : undefined;
290 };
291 const dedupe = ({
292 messages,
293 }: {
294 messages: ReadonlyArray<Readonly<MediaAgentMessage>>;
295 }) => {
296 const deduped = dedupeUserMessageMedia(messages);
297 return deduped ? { messages: deduped } : undefined;
298 };
299 return (agent as SubscribableAgent).subscribe({
300 onRunInitialized: rewrite,
301 onMessagesSnapshotEvent: dedupe,
302 onRunFinalized: dedupe,
303 });
304}
305
306
307function bufferToBase64(
308 buffer: ArrayBuffer,
309 mimeType: string,
310): Promise<string> {
311 return new Promise((resolve, reject) => {
312 const reader = new FileReader();
313 reader.addEventListener(
314 "error",
315 () =>
316 reject(
317 reader.error ?? new Error("The selected media could not be read."),
318 ),
319 { once: true },
320 );
321 reader.addEventListener(
322 "load",
323 () => {
324 if (typeof reader.result !== "string") {
325 reject(new Error("The selected media returned an invalid result."));
326 return;
327 }
328 const comma = reader.result.indexOf(",");
329 resolve(comma >= 0 ? reader.result.slice(comma + 1) : reader.result);
330 },
331 { once: true },
332 );
333 reader.readAsDataURL(new Blob([buffer], { type: mimeType }));
334 });
335}
336
337
338function createMessageId(): string {
339 return (
340 globalThis.crypto?.randomUUID?.() ??
341 `angular-${Date.now()}-${Math.random().toString(16).slice(2)}`
342 );
343}
344