openbot AI coworker with a computer

58async function resolveRequestActor(request: Request): Promise<{
59  id: string;
60  name: string;
61  role: OpenBotRole;
62}> {
63  if (config.devNoAuth) {
64    return { id: DEV_ACTOR.id, name: DEV_ACTOR.email, role: DEV_ACTOR.role };
65  }
66  const session = await auth?.api.getSession({ headers: request.headers });
67  const user = session?.user;
68  if (!user) {
69    throw new Error("A CopilotKit run requires a signed-in user.");
70  }
71  const roles = await roleRepository.rolesForUser(user.id);
72  if (!roles.includes("admin") && !roles.includes("user")) {
73    throw new Error("A CopilotKit run requires an authorized user.");
74  }
75  return {
76    id: user.id,
77    name: user.name ?? user.email ?? user.id,
78    role: roles.includes("admin") ? "admin" : "user",
79  };
80}

302  /**
303   * Decide, record, then act.
304   *
305   * The audit row is written before the action runs, not after it succeeds. An allowed action that
306   * later fails is still part of the audit sequence, and a trail that only contains successes cannot
307   * show that sequence.
308   */
309  async function govern<T>(
310    toolName: string,
311    botId: string,
312    actor: ActionActor,
313    subject: {
314      ref?: string;
315      filePath?: string;
316      targetUrl?: string;
317      key?: string;
318      /** The command a shell call is about to run, so a rule can be written against it. */
319      command?: string;
320      /** The person's Stop, on its way to the browser. See the acting methods below. */
321      signal?: AbortSignal;
322    },
323    run: () => Promise<T>,
324  ): Promise<T> {
325    const { ref, filePath } = subject;
326    const element = resolve(botId, ref);
327    const cached = snapshots.get(botId);
328    // For a navigation the relevant page is the one being opened, not the one already loaded. Using
329    // the cached URL would mean `page.host == "..."` could never match the destination, which is the
330    // only thing a rule about navigation would ever want to say.
331    const pageUrl = subject.targetUrl ?? cached?.url ?? "";
332
333    const intent = intentOf(toolName, subject.key);
334
335    const context: PolicyContext = {
336      tool: { name: toolName },
337      bot: { id: botId },
338      actor: { id: actor.id },
339      page: { url: pageUrl, host: hostOf(pageUrl) },
340      ...(intent ? { intent } : {}),
341      ...(subject.key ? { key: subject.key } : {}),
342      ...(element
343        ? {
344            element: {
345              ref: element.ref,
346              role: element.role,
347              name: element.name,
348              ...(element.type ? { type: element.type } : {}),
349            },
350          }
351        : {}),
352      ...(filePath ? { file: describeFile(filePath) } : {}),
353      ...(subject.command ? { command: subject.command } : {}),
354    };
355
356    const decision = evaluateActionPolicy(options.policy(), context);
357    await write(auditStore, {
358      toolName,
359      botId,
360      actor,
361      element,
362      ref,
363      ...(subject.key ? { key: subject.key } : {}),
364      ...(subject.command ? { command: subject.command } : {}),
365      filePath,
366      pageUrl,
367      decision,
368    });
369    if (!decision.forward) {
370      throw new ActionRefusedError(decision.reason, decision.matched);
371    }
372
373    let result: T;
374    try {
375      result = await run();
376    } catch (error) {
377      /**
378       * A permitted action that did not happen gets its own row.
379       *
380       * Without this the trail lies by omission. The row above says the policy allowed the call, and
381       * a reader takes "allowed" to mean "it happened".
382       *
383       * Writing the decision before acting is still right, because an allowed action may have partial
384       * effects before failing. The failure row records the outcome separately from the policy
385       * decision.
386       */
387      await write(auditStore, {
388        toolName,
389        botId,
390        actor,
391        element,
392        ref,
393        filePath,
394        pageUrl,
395        decision,
396        failure: error instanceof Error ? error.message : "The action failed.",
397      });
398      throw error;
399    }
400    // The element's label, attached on the way out, so the transcript can say what was acted on
401    // instead of quoting a ref. The computer cannot supply this: it knows the ref, and the resolved
402    // snapshot lives here. File calls carry their own path already, so there is nothing to add.
403    return element && result && typeof result === "object"
404      ? { ...result, element: { role: element.role, name: element.name } }
405      : result;

21export type ActionPolicy = {
22  /**
23   * `enforce` blocks. `dry-run` decides and records, and lets everything through.
24   *
25   * Dry-run exists so an operator can write a rule against real traffic and read the audit trail
26   * before it starts refusing anybody's work. A governance feature nobody dares switch on is not a
27   * governance feature.
28   */
29  mode: PolicyMode;
30  /** Evaluated first. Any expression true means refused, whatever `allow` says. */
31  deny: string[];
32  /** Any expression true means permitted. Empty means nothing is permitted. */
33  allow: string[];
34};
35
36/**
37 * The attributes a rule can be written against.
38 *
39 * `element` is resolved by the gateway from the snapshot the server itself fetched, never from what
40 * the caller claimed it was clicking. A policy that decides on an attacker-supplied label is
41 * decoration: the whole point is that "do not click Submit" cannot be evaded by calling it something
42 * else in the request.
43 */
44export type PolicyContext = {
45  tool: { name: string };
46  bot: { id: string };
47  page: { url: string; host: string };
48  actor: { id: string };
49  element?: {
50    ref: string;
51    role: string;
52    name: string;
53    type?: string;
54  };
55  /**
56   * The key a `computer_key` call is about to press.
57   *
58   * Without this, a rule about clicking is bypassed. An agent that meets a deny rule on
59   * clicking "Submit order" will press Enter in the form instead, and the order goes through: the
60   * click is refused and audited, the keypress is allowed, because nothing in the context could tell
61   * one keypress from another.
62   *
63   * A form has two doors and the policy could only see one. Now a rule can say
64   * `tool.name == "computer_key" && key == "Enter"`, and an operator blocking a submit button knows
65   * to block both routes, which the deny example in `.env.example` now does.
66   */
67  key?: string;

217async function snapshotPage(
218  session: BotSession,
219  target: Page,
220): Promise<{
221  snapshotId: number;
222  url: string;
223  title: string;
224  elements: SnapshotElement[];
225  truncated: boolean;
226}> {
227  session.snapshotId += 1;
228  const yaml = await target.ariaSnapshot({ mode: "ai" });
229  return {
230    snapshotId: session.snapshotId,
231    url: target.url(),
232    title: await target.title(),
233    ...parseAriaSnapshot(yaml),
234  };
235}
236
237/**
238 * Resolve a ref to a locator, refusing anything from a superseded snapshot.
239 *
240 * `aria-ref=` is a first-party Playwright selector engine, and it is the same one its MCP server uses.
241 * The generation check here is the caller-facing half; see the note on `snapshotId` for why both exist.
242 */
243function locateRef(
244  session: BotSession,
245  target: Page,
246  ref: string,
247  expectedSnapshotId: number | undefined,
248) {
249  if (
250    expectedSnapshotId !== undefined &&
251    expectedSnapshotId !== session.snapshotId
252  ) {
253    throw new StaleSnapshotError(
254      `That list of elements is out of date: it was taken for snapshot ${expectedSnapshotId} and the page is now at ${session.snapshotId}. Take a new snapshot and use the refs from it.`,
255    );
256  }
257  return target.locator(`aria-ref=${ref}`);
258}
259
260/**
261 * The element, or a refusal that says what to do about it.
262 *
263 * A generation check is not an existence check. A ref from
264 * the current snapshot that names nothing on the page, because a model invented it or because the
265 * page moved on without a new snapshot being taken, passes `locateRef` and then simply waits. The
266 * action times out, and the caller gets a generic failure carrying Playwright's internal call log
267 * instead of the actionable answer: take a fresh snapshot.
268 *
269 * `count()` resolves immediately rather than waiting, so a ref that names nothing is refused in
270 * milliseconds instead of holding the action open for the full timeout.
271 */
272async function resolveRef(
273  session: BotSession,
274  target: Page,
275  ref: string,
276  expectedSnapshotId: number | undefined,
277) {
278  const locator = locateRef(session, target, ref, expectedSnapshotId);
279  if ((await locator.count()) === 0) {
280    throw new StaleSnapshotError(
281      `Nothing on this page has the ref ${ref}. Take a new snapshot and use the refs it returns.`,
282    );
283  }
284  return locator;
285}

56export function createShell(workspaceDir: string) {
57  return {
58    async run(input: {
59      command: string;
60      timeoutMs?: number;
61      signal?: AbortSignal;
62    }): Promise<ShellResult> {
63      const started = Date.now();
64      const timeoutMs = Math.min(
65        input.timeoutMs ?? DEFAULT_TIMEOUT_MS,
66        MAX_TIMEOUT_MS,
67      );
68
69      /*
70       * Through a shell on purpose. A Bot writes `apt-get install -y jq && jq --version`, and pipes,
71       * redirection and `&&` are most of why a shell is useful. Refusing them would leave something
72       * that runs one binary and calls itself a shell.
73       *
74       * This is the argument for the container boundary rather than for string parsing: there is no
75       * safe way to read a command's intent from its text, so nothing here tries. The policy decides
76       * whether this Bot may run commands at all and what they may say; the container decides what a
77       * command can reach.
78       */
79      const child = spawn("/bin/bash", ["-lc", input.command], {
80        cwd: workspaceDir,
81        env: { ...process.env, HOME: workspaceDir },
82      });
83
84      let stdout = "";
85      let stderr = "";
86      let timedOut = false;
87
88      child.stdout.on("data", (chunk) => {
89        stdout += String(chunk);
90      });
91      child.stderr.on("data", (chunk) => {
92        stderr += String(chunk);
93      });
94
95      const timer = setTimeout(() => {
96        timedOut = true;
97        child.kill("SIGKILL");
98      }, timeoutMs);
99
100      // The person's Stop reaches the command, not just the request that started it.
101      const onAbort = () => child.kill("SIGKILL");
102      input.signal?.addEventListener("abort", onAbort, { once: true });
103
104      const exitCode = await new Promise<number>((resolve) => {
105        child.on("close", (code) => resolve(code ?? -1));
106        child.on("error", () => resolve(-1));
107      });
108
109      clearTimeout(timer);
110      input.signal?.removeEventListener("abort", onAbort);
111
112      const out = clamp(stdout);
113      const err = clamp(stderr);
114      return {
115        command: input.command,
116        exitCode,
117        stdout: out.text,
118        stderr: err.text,
119        truncated: out.truncated || err.truncated,
120        timedOut,
121        elapsedMs: Date.now() - started,
122      };
123    },
124  };
125}

72export function createControl(
73  now: () => string = () => new Date().toISOString(),
74) {
75  let state: ControlState = {
76    holder: "bot",
77    since: now(),
78    requested: false,
79  };
80
81  return {
82    /** The current state, as the surface polls it. A copy, so a caller cannot mutate the machine. */
83    get(): ControlState {
84      return { ...state };
85    },
86
87    /**
88     * The Bot asking for help.
89     *
90     * It does not take control: it says it is stuck and why, and a person decides. A Bot that could
91     * hand itself to a human could also hand a human a page they never asked to see.
92     */
93    requestHelp(reason: unknown): ControlState {
94      state = {
95        ...state,
96        requested: true,
97        reason:
98          typeof reason === "string" && reason.trim()
99            ? reason.trim()
100            : "The assistant needs a person to continue.",
101      };
102      return this.get();
103    },
104
105    /** The Bot asking for one value it must not be told, naming the field it goes in. */
106    requestSecret(input: {
107      label?: unknown;
108      ref?: unknown;
109      snapshotId?: unknown;
110    }): ControlState {
111      if (typeof input.ref !== "string" || !input.ref.trim()) {
112        throw new ControlRequestError(
113          "Say which field the value goes in, using a ref from your snapshot.",
114        );
115      }
116      state = {
117        ...state,
118        secretWanted:
119          typeof input.label === "string" && input.label.trim()
120            ? input.label.trim()
121            : "the value this page is asking for",
122        secretRef: input.ref.trim(),
123        secretSnapshotId:
124          typeof input.snapshotId === "number" ? input.snapshotId : undefined,
125      };
126      return this.get();
127    },

86const app = new Hono();
87
88app.use("*", async (context, next) => {
89  // Health is open so an orchestrator can check it without holding the token.
90  if (context.req.path === "/health") return next();
91  if (context.req.header("authorization") !== `Bearer ${token}`) {
92    return context.json({ error: "Unauthorized." }, 401);
93  }
94  return next();
95});
96
97app.get("/health", async (context) =>
98  context.json({ status: "ok", docker: await reachable() }),
99);
100
101/** The Bot id in the path, validated before it becomes any kind of name. */
102function resolve(raw: string) {
103  return namesFor(raw);
104}
105
106app.post("/computers/:botId/ensure", async (context) => {
107  const parsed = resolve(context.req.param("botId"));
108  if (!parsed.ok) return context.json({ error: parsed.reason }, 400);
109
110  try {
111    // Registered before the computer is handed out, so it can prove which Bot it is from its first
112    // request.
113    const identity = await registerEntry(parsed.names);
114
115    const state = await ensure(parsed.names, {
116      image,
117      environment: environmentFor(parsed.names.botId),
118      ...(network ? { network } : {}),
119      ...(runtime ? { runtime } : {}),
120      ...(memoryBytes ? { memoryBytes } : {}),
121      ...(spireSocketVolume ? { spireSocketVolume } : {}),
122    });
123    return context.json({
124      ...state,
125      ...(identity.registered
126        ? { spiffeId: identity.spiffeId }
127        : { identity: identity.reason }),
128    });
129  } catch (error) {
130    if (error instanceof DockerUnavailableError) {
131      return context.json({ error: error.message }, 503);
132    }
133    throw error;
134  }
135});
136
137app.post("/computers/:botId/stop", async (context) => {
138  const parsed = resolve(context.req.param("botId"));
139  if (!parsed.ok) return context.json({ error: parsed.reason }, 400);
140  try {
141    const stopped = await stop(parsed.names);
142    return context.json({ stopped });
143  } catch (error) {
144    if (error instanceof DockerUnavailableError) {
145      return context.json({ error: error.message }, 503);
146    }
147    throw error;
148  }
149});
150
151app.post("/computers/:botId/reset", async (context) => {
152  const parsed = resolve(context.req.param("botId"));
153  if (!parsed.ok) return context.json({ error: parsed.reason }, 400);
154  try {
155    const wasThere = await reset(parsed.names);
156    return context.json({ reset: wasThere });
157  } catch (error) {
158    if (error instanceof DockerUnavailableError) {
159      return context.json({ error: error.message }, 503);
160    }
161    throw error;
162  }
163});
164
165app.get("/computers", async (context) => {
166  try {
167    return context.json({ computers: await listOwned() });
168  } catch (error) {
169    if (error instanceof DockerUnavailableError) {
170      return context.json({ error: error.message }, 503);
171    }
172    throw error;
173  }
174});
175
176serve({ port, fetch: app.fetch, idleTimeout: 120 });

17export async function runConnector(
18  adapter: ConnectorAdapter,
19  persistence: ConnectorPersistence,
20  mode: "sync" | "reconcile" = "sync",
21) {
22  try {
23    const discovered = await adapter.discover({
24      cursor: await persistence.cursor(),
25      mode,
26    });
27    if (persistence.persistBatch) {
28      await persistence.persistBatch(discovered.changes, discovered.nextCursor);
29    } else {
30      if (!persistence.apply || !persistence.commitCursor) {
31        throw new Error(
32          "Connector persistence must support batch or individual writes.",
33        );
34      }
35      for (const change of discovered.changes) await persistence.apply(change);
36      if (discovered.nextCursor !== null) {
37        await persistence.commitCursor(discovered.nextCursor);
38      }
39    }
40    await persistence.recordRun?.("succeeded");
41  } catch (error) {
42    await persistence.recordRun?.("failed");
43    throw error;
44  }
45}

210describe("the computer gateway", () => {
211  test("carries out an allowed action and records it", async () => {
212    const { gateway, calls, rows } = await gatewayWith(PERMISSIVE);
213    await gateway.click("bot-1", ACTOR, {
214      ref: "e9",
215      snapshotId: 7,
216    });
217
218    expect(calls).toEqual(["click"]);
219    expect(rows).toHaveLength(1);
220    expect(rows[0]?.eventType).toBe("computer.action_allowed");
221    expect(rows[0]?.targetId).toBe("bot-1");
222  });
223
224  test("a refused action never reaches the computer", async () => {
225    const { gateway, calls, rows } = await gatewayWith({
226      ...PERMISSIVE,
227      deny: ['contains(element.name, "submit")'],
228    });
229
230    await expect(
231      gateway.click("bot-1", ACTOR, { ref: "e9", snapshotId: 7 }),
232    ).rejects.toThrow(ActionRefusedError);
233
234    // The decision happens before the effect.
235    expect(calls).toEqual([]);
236    expect(rows).toHaveLength(1);
237    expect(rows[0]?.eventType).toBe("computer.action_refused");
238    expect(rows[0]?.targetId).toBe("bot-1");
239  });
240
241  test("the refusal names the rule, so an operator can find it", async () => {
242    const { gateway } = await gatewayWith({
243      ...PERMISSIVE,
244      deny: ['contains(element.name, "submit")'],
245    });
246
247    const error = await gateway
248      .click("bot-1", ACTOR, { ref: "e9", snapshotId: 7 })
249      .catch((caught: unknown) => caught);
250
251    expect(error).toBeInstanceOf(ActionRefusedError);
252    expect((error as ActionRefusedError).rule).toBe(
253      'contains(element.name, "submit")',
254    );
255  });
256
257  test("the policy sees the element the SERVER resolved, not what the caller claimed", async () => {
258    // The evasion this prevents: calling the Submit button something harmless in the request. The
259    // caller only ever sends a ref, and the gateway looks it up in the snapshot it fetched itself.
260    const { gateway, calls } = await gatewayWith({
261      ...PERMISSIVE,
262      deny: ['contains(element.name, "submit")'],
263    });
264
265    await expect(
266      gateway.click("bot-1", ACTOR, {
267        ref: "e9",
268        snapshotId: 7,
269        // Not part of the input contract, and must not influence anything even when supplied.
270        ...({ name: "Continue", element: { name: "Continue" } } as object),
271      }),
272    ).rejects.toThrow(ActionRefusedError);
273    expect(calls).toEqual([]);
274  });