1import type {
2  AttachmentsConfig,
3  RenderToolCallConfig,
4} from "@copilotkit/angular";
5import {
6  ChangeDetectionStrategy,
7  Component,
8  effect,
9  inject,
10  signal,
11  viewChild,
12} from "@angular/core";
13import { ActivatedRoute } from "@angular/router";
14import {
15  CopilotKit,
16  injectAgentStore,
17  registerRenderToolCall,
18} from "@copilotkit/angular";
19import { z } from "zod";
20
21import { agentIdForCurrentIntegration } from "../../feature-agent";
22import { FeatureHeaderComponent } from "../feature-header.component";
23import { ShowcaseChatHostComponent } from "../showcase-chat-host.component";
24import { WeatherToolCard } from "../tools/tool-cards";
25import {
26  createMultimodalMessage,
27  dedupeUserMessageMedia,
28  validateSampleBytes,
29  VOICE_WEATHER_TOOL_NAMES,
30} from "./media-model";
31import type { MediaAgentMessage, SampleSpec } from "./media-model";
32
33const VOICE_SAMPLE_TEXT = "What is the weather in Tokyo?";
34
35type VoiceWeatherArgs = Record<string, unknown>;
36
37/** Exact backend weather tools rendered by the voice demo. */
38export const voiceWeatherRendererConfigs: readonly RenderToolCallConfig<VoiceWeatherArgs>[] =
39  VOICE_WEATHER_TOOL_NAMES.map((name) => ({
40    name,
41    args: z.record(z.unknown()),
42    component:
43      WeatherToolCard as unknown as RenderToolCallConfig<VoiceWeatherArgs>["component"],
44  }));
45
46const SAMPLES: readonly SampleSpec[] = [
47  {
48    buttonLabel: "Try with sample image",
49    filename: "sample.png",
50    mimeType: "image/png",
51    testId: "multimodal-sample-image-button",
52    fetchUrl: "/demo-files/sample.png",
53    autoPrompt: "can you tell me what is in this demo image I just attached",
54  },
55  {
56    buttonLabel: "Try with sample PDF",
57    filename: "sample.pdf",
58    mimeType: "application/pdf",
59    testId: "multimodal-sample-pdf-button",
60    fetchUrl: "/demo-files/sample.pdf",
61    autoPrompt: "can you tell me what is in this demo pdf I just attached",
62  },
63];
64
65const MULTIMODAL_ATTACHMENTS: AttachmentsConfig = {
66  enabled: true,
67  accept: "image/*,application/pdf",
68  maxSize: 10 * 1024 * 1024,
69};
70
71@Component({
72  selector: "showcase-media-feature",
73  imports: [FeatureHeaderComponent, ShowcaseChatHostComponent],
74  changeDetection: ChangeDetectionStrategy.OnPush,
75  host: { class: "feature-page" },
76  template: `
77    <showcase-feature-header />
78    <main class="media-page">
79      <section class="sample-panel" aria-label="Bundled media samples">
80        @if (feature === "voice") {
81          <div class="sample-copy">
82            <strong>Voice input</strong>
83            <span>Use the microphone, or insert a deterministic sample.</span>
84          </div>
85          <button
86            type="button"
87            data-testid="voice-sample-audio-button"
88            [title]="'Inserts: &quot;' + voiceSampleText + '&quot;'"
89            (click)="insertVoiceSample()"
90          >
91            <span aria-hidden="true">🎙</span>
92            <span>Try a sample audio</span>
93          </button>
94        } @else {
95          <div class="sample-copy">
96            <strong>Bundled samples</strong>
97            <span>Send a real image or PDF through the agent runtime.</span>
98          </div>
99          <div class="sample-actions">
100            @for (sample of samples; track sample.testId) {
101              <button
102                type="button"
103                [attr.data-testid]="sample.testId"
104                [disabled]="loading() !== null"
105                (click)="sendSample(sample)"
106              >
107                {{ loading() === sample.testId ? "Sending…" : sample.buttonLabel }}
108              </button>
109            }
110          </div>
111        }
112        @if (error()) {
113          <p class="sample-error" role="alert">{{ error() }}</p>
114        }
115      </section>
116      <section class="chat-surface" aria-label="CopilotKit assistant">
117        <showcase-chat-host
118          [agentId]="agentId"
119          [attachments]="
120            feature === 'multimodal' ? multimodalAttachments : undefined
121          "
122        />
123      </section>
124    </main>
125  `,
126  styles: `
127    .media-page {
128      display: grid;
129      min-height: 0;
130      grid-template-rows: auto minmax(0, 1fr);
131      gap: 0.75rem;
132      padding: 0.75rem;
133      background: light-dark(#eef3f7, #0a0a0a);
134    }
135    .sample-panel {
136      display: flex;
137      flex-wrap: wrap;
138      align-items: center;
139      gap: 0.75rem 1rem;
140      padding: 0.75rem 1rem;
141      border: 1px solid light-dark(#d8e0ea, #303136);
142      border-radius: 0.9rem;
143      background: light-dark(#fff, #191a1e);
144    }
145    .sample-copy {
146      display: grid;
147      flex: 1 1 16rem;
148      gap: 0.15rem;
149      color: light-dark(#14213d, #fafafa);
150    }
151    .sample-copy span {
152      color: light-dark(#52637a, #adadb2);
153      font-size: 0.82rem;
154    }
155    .sample-actions {
156      display: flex;
157      flex-wrap: wrap;
158      gap: 0.5rem;
159    }
160    button {
161      display: inline-flex;
162      align-items: center;
163      gap: 0.4rem;
164      padding: 0.55rem 0.8rem;
165      border: 1px solid light-dark(#c7d2e0, #303136);
166      border-radius: 0.65rem;
167      color: light-dark(#20324d, #fafafa);
168      background: light-dark(#fff, #191a1e);
169      font: inherit;
170      font-size: 0.84rem;
171      font-weight: 650;
172      cursor: pointer;
173    }
174    button:hover:not(:disabled) {
175      border-color: #4263eb;
176      background: light-dark(#f5f7ff, rgb(30 27 75 / 40%));
177    }
178    button:focus-visible {
179      outline: 3px solid #91a7ff;
180      outline-offset: 2px;
181    }
182    button:disabled {
183      cursor: wait;
184      opacity: 0.62;
185    }
186    .sample-error {
187      flex-basis: 100%;
188      margin: 0;
189      color: light-dark(#991b1b, #fca5a5);
190      font-size: 0.84rem;
191    }
192    .chat-surface {
193      min-height: 0;
194      overflow: hidden;
195      border: 1px solid light-dark(#d8e0ea, #303136);
196      border-radius: 1rem;
197      background: light-dark(#fff, #191a1e);
198    }
199  `,
200})
201export class MediaFeatureComponent {
202  private readonly route = inject(ActivatedRoute);
203  private readonly copilotKit = inject(CopilotKit);
204  private readonly chatHost = viewChild.required(ShowcaseChatHostComponent);
205  protected readonly feature =
206    (this.route.snapshot.data["feature"] as string | undefined) ?? "voice";
207  protected readonly agentId = agentIdForCurrentIntegration(this.feature);
208  private readonly agentStore = injectAgentStore(this.agentId);
209  protected readonly voiceSampleText = VOICE_SAMPLE_TEXT;
210  protected readonly samples = SAMPLES;
211  protected readonly multimodalAttachments = MULTIMODAL_ATTACHMENTS;
212  protected readonly loading = signal<string | null>(null);
213  protected readonly error = signal<string | null>(null);
214
215  constructor() {
216    if (this.feature === "voice") {
217      for (const config of voiceWeatherRendererConfigs) {
218        registerRenderToolCall(config);
219      }
220    }
221    if (this.feature === "multimodal") {
222      effect((onCleanup) => {
223        const handle = installMediaDedupe(this.agentStore().agent);
224        onCleanup(() => handle.unsubscribe());
225      });
226    }
227  }
228
229  /** Insert the deterministic voice sample into the actual chat composer. */
230  protected insertVoiceSample(): void {
231    if (!this.chatHost().populateComposer(VOICE_SAMPLE_TEXT)) {
232      this.error.set("The chat composer is not ready yet. Try again.");
233      return;
234    }
235    this.error.set(null);
236  }
237
238  /** Fetch, validate, attach, and dispatch one canonical media sample. */
239  protected async sendSample(spec: SampleSpec): Promise<void> {
240    if (this.loading() !== null) return;
241    this.loading.set(spec.testId);
242    this.error.set(null);
243    try {
244      const response = await fetch(spec.fetchUrl);
245      if (!response.ok) {
246        throw new Error(
247          `Could not fetch sample "${spec.filename}" (HTTP ${response.status}).`,
248        );
249      }
250      const buffer = await response.arrayBuffer();
251      const bytes = new Uint8Array(buffer);
252      validateSampleBytes(bytes, spec.mimeType, spec.filename);
253      const base64 = await bufferToBase64(buffer, spec.mimeType);
254      const agent = this.agentStore().agent;
255      const message = createMultimodalMessage(
256        spec,
257        base64,
258        buffer.byteLength,
259        createMessageId(),
260      );
261      agent.addMessage(message as Parameters<typeof agent.addMessage>[0]);
262      await this.copilotKit.core.runAgent({ agent });
263    } catch (error: unknown) {
264      console.error("[showcase-angular:multimodal] Sample send failed", error);
265      this.error.set(
266        error instanceof Error ? error.message : "The sample send failed.",
267      );
268    } finally {
269      this.loading.set(null);
270    }
271  }
272}
273
274interface SubscribableAgent {
275  subscribe: (subscriber: unknown) => { unsubscribe: () => void };
276}
277
278/**
279 * Install the inbound media dedupe subscribers on one active agent.
280 *
281 * Outgoing messages are left as modern AG-UI media parts. Do not append a
282 * legacy `binary` mirror: AG-UI protocol 1.0 removed that part, and Python
283 * bridges on 1.0 reject the whole run with HTTP 422.
284 */
285function installMediaDedupe(agent: object): {
286  unsubscribe: () => void;
287} {
288  const dedupe = ({
289    messages,
290  }: {
291    messages: ReadonlyArray<Readonly<MediaAgentMessage>>;
292  }) => {
293    const deduped = dedupeUserMessageMedia(messages);
294    return deduped ? { messages: deduped } : undefined;
295  };
296  return (agent as SubscribableAgent).subscribe({
297    onMessagesSnapshotEvent: dedupe,
298    onRunFinalized: dedupe,
299  });
300}
301
302/** Convert browser bytes to the base64 payload used by AG-UI data sources. */
303function bufferToBase64(
304  buffer: ArrayBuffer,
305  mimeType: string,
306): Promise<string> {
307  return new Promise((resolve, reject) => {
308    const reader = new FileReader();
309    reader.addEventListener(
310      "error",
311      () =>
312        reject(
313          reader.error ?? new Error("The selected media could not be read."),
314        ),
315      { once: true },
316    );
317    reader.addEventListener(
318      "load",
319      () => {
320        if (typeof reader.result !== "string") {
321          reject(new Error("The selected media returned an invalid result."));
322          return;
323        }
324        const comma = reader.result.indexOf(",");
325        resolve(comma >= 0 ? reader.result.slice(comma + 1) : reader.result);
326      },
327      { once: true },
328    );
329    reader.readAsDataURL(new Blob([buffer], { type: mimeType }));
330  });
331}
332
333/** Create a collision-resistant client message identifier. */
334function createMessageId(): string {
335  return (
336    globalThis.crypto?.randomUUID?.() ??
337    `angular-${Date.now()}-${Math.random().toString(16).slice(2)}`
338  );
339}
340
7fdfe25e2
CopilotKit Showcase