Add Tzafon Northstar computer use template

pkg/create/templates.go

@@ -19,6 +19,7 @@ const (
 	TemplateOpenAGIComputerUse   = "openagi-computer-use"
 	TemplateClaudeAgentSDK       = "claude-agent-sdk"
 	TemplateYutoriComputerUse    = "yutori"
+	TemplateTzafonComputerUse    = "tzafon-computer-use"
 )
 
 type TemplateInfo struct {
@@ -90,6 +91,11 @@ var Templates = map[string]TemplateInfo{
 		Description: "Implements a Yutori n1 computer use agent",
 		Languages:   []string{LanguageTypeScript, LanguagePython},
 	},
+	TemplateTzafonComputerUse: {
+		Name:        "Tzafon Northstar Computer Use",
+		Description: "Implements a Tzafon Northstar CUA Fast computer use agent",
+		Languages:   []string{LanguageTypeScript, LanguagePython},
+	},
 }
 
 // GetSupportedTemplatesForLanguage returns a list of all supported template names for a given language
@@ -116,6 +122,8 @@ func GetSupportedTemplatesForLanguage(language string) TemplateKeyValues {
 				return 2
 			case TemplateYutoriComputerUse:
 				return 3
+			case TemplateTzafonComputerUse:
+				return 4
 			default:
 				return 10
 			}
@@ -213,6 +221,11 @@ var Commands = map[string]map[string]DeployConfig{
 			NeedsEnvFile:  true,
 			InvokeCommand: `kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com and describe the page"}'`,
 		},
+		TemplateTzafonComputerUse: {
+			EntryPoint:    "index.ts",
+			NeedsEnvFile:  true,
+			InvokeCommand: `kernel invoke ts-tzafon-cua cua-task --payload '{"query": "Go to wikipedia.org and search for Alan Turing"}'`,
+		},
 	},
 	LanguagePython: {
 		TemplateSampleApp: {
@@ -260,6 +273,11 @@ var Commands = map[string]map[string]DeployConfig{
 			NeedsEnvFile:  true,
 			InvokeCommand: `kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com and describe the page"}'`,
 		},
+		TemplateTzafonComputerUse: {
+			EntryPoint:    "main.py",
+			NeedsEnvFile:  true,
+			InvokeCommand: `kernel invoke python-tzafon-cua cua-task --payload '{"query": "Go to wikipedia.org and search for Alan Turing"}'`,
+		},
 	},
 }
 

pkg/templates/python/tzafon-computer-use/.env.example

@@ -0,0 +1 @@
+TZAFON_API_KEY=your-tzafon-api-key

pkg/templates/python/tzafon-computer-use/README.md

@@ -0,0 +1,60 @@
+# Kernel Python Sample App - Tzafon Northstar Computer Use
+
+This is a Kernel application that implements a CUA (computer use agent) loop using Tzafon's Northstar CUA Fast model with Kernel's Computer Controls API. The model is accessed via Tzafon's [Lightcone](https://docs.lightcone.ai) API platform.
+
+[Northstar CUA Fast](https://docs.lightcone.ai) is a 4B-parameter vision model trained with reinforcement learning for computer use tasks.
+
+## Setup
+
+1. Get your API keys:
+   - **Kernel**: [dashboard.onkernel.com](https://dashboard.onkernel.com)
+   - **Tzafon**: [tzafon.ai](https://www.tzafon.ai)
+
+2. Deploy the app:
+```bash
+kernel login
+cp .env.example .env  # Add your TZAFON_API_KEY
+kernel deploy main.py --env-file .env
+```
+
+## Usage
+
+```bash
+kernel invoke python-tzafon-cua cua-task --payload '{"query": "Go to wikipedia.org and search for Alan Turing"}'
+```
+
+## Recording Replays
+
+> **Note:** Replay recording is only available to Kernel users on paid plans.
+
+Add `"record_replay": true` to your payload to capture a video of the browser session:
+
+```bash
+kernel invoke python-tzafon-cua cua-task --payload '{"query": "Navigate to https://example.com", "record_replay": true}'
+```
+
+When enabled, the response will include a `replay_url` field with a link to view the recorded session.
+
+## Viewport Configuration
+
+Northstar CUA Fast works well with a **1280x800** viewport, which is the default.
+
+## Supported Actions
+
+| Action | Description |
+|--------|-------------|
+| `click` | Left mouse click at coordinates |
+| `double_click` | Double-click at coordinates |
+| `triple_click` | Triple-click at coordinates |
+| `right_click` | Right mouse click at coordinates |
+| `scroll` | Scroll page vertically or horizontally |
+| `type` | Type text into focused element |
+| `key` / `keypress` | Send keyboard input |
+| `navigate` | Navigate to a URL |
+| `drag` | Click-and-drag operation |
+| `wait` | Pause for UI to update |
+
+## Resources
+
+- [Lightcone API Documentation](https://docs.lightcone.ai)
+- [Kernel Documentation](https://www.kernel.sh/docs/quickstart)

pkg/templates/python/tzafon-computer-use/_gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.py[cod]
+*$py.class
+.env
+*.log
+.venv/
+venv/

pkg/templates/python/tzafon-computer-use/loop.py

@@ -0,0 +1,124 @@
+"""
+Tzafon Northstar Sampling Loop
+
+Implements the agent loop for Tzafon's Northstar CUA Fast model.
+Northstar uses the Lightcone SDK with a responses API:
+- Actions are returned via computer_call outputs
+- Tool results use computer_call_output with screenshot images
+- The model stops when no computer_call is in the output or the action is terminal
+- Continuation uses previous_response_id for multi-turn context
+
+@see https://docs.lightcone.ai
+"""
+
+import asyncio
+from typing import Any, Optional
+
+from kernel import Kernel
+from tzafon import Lightcone
+
+from tools import ComputerTool
+
+MODEL = "tzafon.northstar-cua-fast"
+
+TOOL = {
+    "type": "computer_use",
+    "display_width": 1280,
+    "display_height": 800,
+    "environment": "browser",
+}
+
+TERMINAL_ACTIONS = {"terminate", "done", "answer"}
+
+
+async def sampling_loop(
+    *,
+    task: str,
+    api_key: str,
+    kernel: Kernel,
+    session_id: str,
+    model: str = MODEL,
+    max_steps: int = 50,
+    viewport_width: int = 1280,
+    viewport_height: int = 800,
+) -> dict[str, Any]:
+    """Run the Northstar CUA loop until the model terminates or max steps are reached."""
+    tzafon = Lightcone(api_key=api_key)
+    computer = ComputerTool(kernel, session_id, viewport_width, viewport_height)
+
+    tool = {
+        **TOOL,
+        "display_width": viewport_width,
+        "display_height": viewport_height,
+    }
+
+    screenshot_url = computer.capture_screenshot()
+
+    response = tzafon.responses.create(
+        model=model,
+        tools=[tool],
+        input=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_text", "text": task},
+                    {"type": "input_image", "image_url": screenshot_url},
+                ],
+            }
+        ],
+    )
+
+    final_result: Optional[str] = None
+
+    for step in range(max_steps):
+        computer_call = next(
+            (o for o in (response.output or []) if o.type == "computer_call"),
+            None,
+        )
+        if not computer_call:
+            break
+
+        action = computer_call.action
+        label = action.type
+        if action.x is not None:
+            label += f" @ ({action.x}, {action.y})"
+        if action.text:
+            label += f" '{action.text}'"
+        print(f"[{step + 1}] {label}")
+
+        if action.type in TERMINAL_ACTIONS:
+            final_result = action.result or action.text or action.status
+            print(f"Result: {final_result}")
+            break
+
+        await computer.execute(action)
+        await asyncio.sleep(1)
+
+        screenshot_url = computer.capture_screenshot()
+        response = tzafon.responses.create(
+            model=model,
+            previous_response_id=response.id,
+            tools=[tool],
+            input=[
+                {
+                    "type": "computer_call_output",
+                    "call_id": computer_call.call_id,
+                    "output": {
+                        "type": "input_image",
+                        "image_url": screenshot_url,
+                    },
+                }
+            ],
+        )
+
+    messages: list[str] = []
+    for item in response.output or []:
+        if item.type == "message":
+            for block in item.content or []:
+                if block.text:
+                    messages.append(block.text)
+
+    return {
+        "messages": messages,
+        "final_result": final_result,
+    }

pkg/templates/python/tzafon-computer-use/main.py

@@ -0,0 +1,65 @@
+import os
+from typing import Optional, TypedDict
+
+import kernel
+from loop import sampling_loop
+from session import KernelBrowserSession
+
+
+class QueryInput(TypedDict):
+    query: str
+    record_replay: Optional[bool]
+
+
+class QueryOutput(TypedDict):
+    result: str
+    replay_url: Optional[str]
+
+
+api_key = os.getenv("TZAFON_API_KEY")
+if not api_key:
+    raise ValueError("TZAFON_API_KEY is not set")
+
+app = kernel.App("python-tzafon-cua")
+
+
+@app.action("cua-task")
+async def cua_task(
+    ctx: kernel.KernelContext,
+    payload: QueryInput,
+) -> QueryOutput:
+    if not payload or not payload.get("query"):
+        raise ValueError("Query is required")
+
+    record_replay = payload.get("record_replay", False)
+
+    async with KernelBrowserSession(
+        invocation_id=ctx.invocation_id,
+        stealth=True,
+        record_replay=record_replay,
+    ) as session:
+        print("Kernel browser live view url:", session.live_view_url)
+
+        loop_result = await sampling_loop(
+            task=payload["query"],
+            api_key=str(api_key),
+            kernel=session.kernel,
+            session_id=str(session.session_id),
+            viewport_width=session.viewport_width,
+            viewport_height=session.viewport_height,
+        )
+
+        final_result = loop_result.get("final_result")
+        messages = loop_result.get("messages", [])
+
+        if final_result:
+            result = final_result
+        elif messages:
+            result = messages[-1]
+        else:
+            result = "Task completed"
+
+    return {
+        "result": result,
+        "replay_url": session.replay_view_url,
+    }

pkg/templates/python/tzafon-computer-use/pyproject.toml

@@ -0,0 +1,9 @@
+[project]
+name = "python-tzafon-cua"
+version = "0.1.0"
+description = "Kernel reference app for Tzafon Northstar Computer Use"
+requires-python = ">=3.11"
+dependencies = [
+    "kernel>=0.35.0",
+    "tzafon",
+]