From fcb673b14e7488efbd01964c2cdf57cfc7b7c211 Mon Sep 17 00:00:00 2001 From: Harry Chen Date: Sun, 20 Sep 2026 13:52:49 -0400 Subject: [PATCH] add MCP server + GUI editing plan --- .mcp.json | 8 + AGENTS.md | 4 + cmake/SsApps.cmake | 4 + docs/README.md | 2 + docs/notes/gui-automation.md | 148 ++++++ docs/notes/gui-editing-plan.md | 358 +++++++++++++++ src/app/gui/Automation.cpp | 804 +++++++++++++++++++++++++++++++++ src/app/gui/Automation.h | 40 ++ src/app/gui/GuiApp.cpp | 30 ++ src/app/gui/GuiApp.h | 5 + src/app/gui/GuiMain.cpp | 16 +- tools/gui_mcp.py | 302 +++++++++++++ tools/guictl.py | 288 ++++++++++++ 13 files changed, 2008 insertions(+), 1 deletion(-) create mode 100644 .mcp.json create mode 100644 docs/notes/gui-automation.md create mode 100644 docs/notes/gui-editing-plan.md create mode 100644 src/app/gui/Automation.cpp create mode 100644 src/app/gui/Automation.h create mode 100755 tools/gui_mcp.py create mode 100755 tools/guictl.py diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 00000000..33bb5e19 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "spirula-gui": { + "command": "python3", + "args": ["tools/gui_mcp.py"] + } + } +} diff --git a/AGENTS.md b/AGENTS.md index 99dc7582..86c21ff6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,10 @@ assets/fonts/ the five embedded UI faces + the full-CJK face table (assets/fonts/README.md). The CJK subsets are GENERATED from the catalogs -- see below tools/codegen/ the four codegen tools (see "Codegen" below) +tools/guictl.py drive the GUI from a script -- list the widgets on + screen, click them, read the framebuffer back. + tools/gui_mcp.py is the same surface as an MCP + server; docs/notes/gui-automation.md reference/scripts/ dataset preprocessing CLI tools (Python, standalone; mask.py is embedded into the GUI binary) reference/python/ hand-run tools on NO code path: eval_lpips.py, diff --git a/cmake/SsApps.cmake b/cmake/SsApps.cmake index 9c6e69fe..27fc8601 100644 --- a/cmake/SsApps.cmake +++ b/cmake/SsApps.cmake @@ -170,6 +170,10 @@ if(SS_BUILD_GUI) ) target_compile_options(imgui_glfw PRIVATE $<$:${SPLAT_CXX_FLAGS}>) + # The item hooks src/app/gui/Automation.cpp implements: one never-taken + # branch per widget until it arms them. Not an option -- imgui references + # the hooks once this is on, so a build without Automation.cpp would fail. + target_compile_definitions(imgui_glfw PUBLIC IMGUI_ENABLE_TEST_ENGINE) target_include_directories(imgui_glfw PUBLIC ${imgui_SOURCE_DIR} ${imgui_SOURCE_DIR}/backends diff --git a/docs/README.md b/docs/README.md index 911d1100..ffdcc0c3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,8 @@ the detail. | [notes/compare-view.md](notes/compare-view.md) | showing several models at once: engine scene slots, the shared navigation frame | | [notes/vram-splat-x-img.md](notes/vram-splat-x-img.md) | what the largest scratch category costs per element, the bitmask compaction, and the measured dead ends | | [notes/color-transfer.md](notes/color-transfer.md) | linear storage vs the output tone curve, the dynamic range a curve buys, and why its clip is straight-through | +| [notes/gui-editing-plan.md](notes/gui-editing-plan.md) | editing in the GUI: the selection seam every tool shares, transforms, mask editing, trajectories, and the order to build them in | +| [notes/gui-automation.md](notes/gui-automation.md) | driving the GUI from a script: the imgui item hooks, the loopback control surface, `tools/guictl.py` and the MCP server | | [notes/sfm-in-process-plan.md](notes/sfm-in-process-plan.md) | running SfM inside the GUI: the library seam, the input manifest, and what manual ties need | | [notes/](notes/) | design notes for individual subsystems | diff --git a/docs/notes/gui-automation.md b/docs/notes/gui-automation.md new file mode 100644 index 00000000..c280a2cf --- /dev/null +++ b/docs/notes/gui-automation.md @@ -0,0 +1,148 @@ +# Driving the GUI from outside it + +`src/app/gui/Automation.{h,cpp}` opens a loopback HTTP surface that lists the +widgets currently on screen, injects mouse and keyboard input at them, and +hands back the framebuffer as an image. `tools/guictl.py` is the shell client +and `tools/gui_mcp.py` the same thing as an MCP server. + +It exists because the editing work +([gui-editing-plan.md](gui-editing-plan.md)) is interaction code — a drag has +to produce the selection the user meant — and the only way to check that used +to be a human at the keyboard. It is also the cheapest regression test this +GUI has ever had: a script and a screenshot. + +## Why not the usual approaches + +Screen automation from outside the process (`xdotool` and friends) needs a real +display server, knows nothing about widgets, and on Wayland is mostly blocked +by design. Dear ImGui's own test engine does all of this properly, but it is a +separate dependency under its own licence, and the part of it that matters here +is three functions. + +Those three functions are the ones ImGui already calls: build with +`IMGUI_ENABLE_TEST_ENGINE` and `ItemAdd()` reports every widget's id and +rectangle, the widgets report their label and status flags, and the hooks are +ours to implement. That is the whole item registry, from four symbols and no +call-site changes. + +It costs one never-taken branch per widget per frame while +`ctx->TestEngineHookItems` is false, which is every run that did not ask for +this. + +## Addressing a widget + +The GUI hands ImGui `"text###"` for everything with an id +(`src/app/gui/Ui.h`), so that a language switch does not change a widget's +identity and collapse every open header. The same property is what makes +scripting it pleasant: the part after `###` is the i18n message name, it is the +same in all thirteen languages, and it is a name a person can read. + +``` +$ python3 tools/guictl.py tree -q open +home_open_dataset Open a Dataset... 800,264 ##host/##home_... +home_open_splat View a Trained Model 800,348 ##host/##home_... +$ python3 tools/guictl.py click home_open_splat +``` + +Widgets ImGui was given a hidden label (`"##fov"`) are addressable under that +spelling. Anything with no id at all — plain text, custom drawing, the viewport +image — is reached by coordinates instead, which is also how the viewport +gestures are driven. + +## How a click happens + +Injecting input is not one event. ImGui trickles its input queue so that a +mouse move and the button press after it land on *different* frames, and an +item is only hovered on the frame after the pointer reaches it. So a command +expands into a short script of steps, and the frame loop applies exactly one +step per frame: move, move, press, release, settle. + +`begin_frame()` runs between `ImGui_ImplGlfw_NewFrame()` and +`ImGui::NewFrame()` — after the backend's own mouse update, so an injected +position is the later event and wins. `end_frame()` runs after the draw data is +rendered and before the buffers are swapped, which is where the item table for +the finished frame is published and where a screenshot reads the back buffer. + +An HTTP request blocks until its last step has been applied and two more frames +have been drawn, so a command returns when its effect is on screen. `nowait=1` +opts out. + +Two smaller details, both of which were bugs first: `ConfigDebugIgnoreFocusLoss` +is set, because a driven window is rarely the focused one and ImGui otherwise +clears the input state it was just handed; and a frame that applied a step +counts as busy, so a scripted run paces at the busy frame rate instead of the +15 Hz idle one. + +## Running it + +```bash +python3 tools/guictl.py launch --offscreen # or: SS_GUI_AUTOMATION=1 ./build_cuda/spirula +python3 tools/guictl.py state +python3 tools/guictl.py tree -q mesh +python3 tools/guictl.py click menu_file +python3 tools/guictl.py key Escape +python3 tools/guictl.py drag 800,600 1000,550 # orbit the viewport +python3 tools/guictl.py text '##fov' 120 +python3 tools/guictl.py shot out.png --width 1280 --format jpg +python3 tools/guictl.py script my-run.txt # one command per line +``` + +`launch` passes any trailing arguments to the binary, which takes file paths +exactly as a drag-and-drop would — the shortest way to get a model open. + +| variable | | +|---|---| +| `SS_GUI_AUTOMATION=1` | arm it; without this nothing binds and no hook runs | +| `SS_GUI_AUTOMATION_PORT` | default 7777 | +| `SS_GUI_AUTOMATION_TOKEN` | fixed token instead of a fresh random one | +| `SS_GUI_OFFSCREEN=1` | create the window invisible (still a real GL context) | + +The server binds `127.0.0.1` only and requires a token. That is not paranoia +about the network: these are GETs with side effects, and any page in any +browser can issue one at localhost. The token is written to +`/automation.json`, which is where both clients read it from. + +Offscreen still needs a display to create a context against. For a machine with +none, run it under `Xvfb`; the rest behaves identically. + +## Endpoints + +All GET, all under `/ui/`, all taking `token=`. `id=` names a widget and +`at=x,y` a point in ImGui display coordinates; `index=` picks between items +sharing an id. + +| | | +|---|---| +| `/ui/state` | frame counter, item count, queue depth, display and framebuffer size, plus what `GuiApp::state_json()` reports: screen, training phase and step, whether a dialog is open, how many models are loaded | +| `/ui/tree` | the widgets of the last finished frame: `q=` substring, `window=`, `named=0` to include unnamed items | +| `/ui/click` | `button=` 0/1/2, `double=1`, `settle=` | +| `/ui/move` | hover, for tooltips and hover-only state | +| `/ui/drag` | `from=`, `to=`, `steps=` — more steps for a path a tool has to follow | +| `/ui/scroll` | `dy=` | +| `/ui/key` | `keys=Ctrl+Shift+A` | +| `/ui/text` | focus, select all, type `value=`, `enter=0` to leave it open | +| `/ui/wait` | `frames=` | +| `/ui/screenshot` | `width=` to downscale (area average), `format=jpg`, `quality=`, `path=` to write it server-side | + +Display coordinates are not framebuffer pixels on a HiDPI screen; `/ui/state` +reports both and the scale between them. + +## As an MCP server + +`tools/gui_mcp.py` wraps the same endpoints as MCP tools over stdio, in the +standard library alone — no dependency, and MCP over stdio is newline-delimited +JSON-RPC. `.mcp.json` at the repo root registers it for Claude Code; anything +else wants `python3 tools/gui_mcp.py` as the command. + +The reason to prefer it over the shell client is `gui_screenshot`, which +returns the picture as an image rather than as a path to one. It downscales to +1280 and encodes JPEG by default: the full-size PNG is two megabytes of base64, +which costs far more than the picture is worth. + +## What it is not + +It drives the *interface*. It does not know what a tool means, and a test that +asserts "this drag selected 412 splats" wants the op-script path in +[gui-editing-plan.md](gui-editing-plan.md) instead, which needs no window at +all. Use this for the part that genuinely needs a window: that the gesture +reaches the tool, that the panel is laid out, that the picture is right. diff --git a/docs/notes/gui-editing-plan.md b/docs/notes/gui-editing-plan.md new file mode 100644 index 00000000..36cb5625 --- /dev/null +++ b/docs/notes/gui-editing-plan.md @@ -0,0 +1,358 @@ +# Editing in the GUI + +A plan for the editing features users keep asking for: placing a model, +selecting parts of one and doing something to the selection, cleaning up a +model by attribute, fixing a mask by hand, laying out a camera move, and +splitting a reconstruction that came out as one model but is two. + +Nothing here is a research problem. Every one of these features is a solved +interaction somewhere else — Blender, Photoshop, CloudCompare, MeshLab — and +the work is deciding what they share so that the eighth one is cheap rather +than an eighth of the code again. + +## The mistake to avoid + +The request reads as seven features. Built as seven features it is seven tool +modes, seven undo stories, seven sets of keys, seven translations, and seven +places where a later feature does not compose with an earlier one — the point +where "select the blue splats *inside this box*" turns out to need a rewrite +because the box tool and the colour tool each own a private answer. + +Almost all of it is the same two things over four kinds of document: + +**make a set**, and **do something to the set**. + +Today's list is about ten ways to make a set and about ten things to do with +one. As features that is a hundred; across a seam it is twenty, and the +twenty-first is a day's work. Everything below follows from taking that seam +seriously. + +## The four documents + +| document | element | what already exists | +|---|---|---| +| sparse reconstruction | 3D point, image, submodel | `ParsedDataset`, `src/sfm/` (incl. `map/Merge.h`) | +| splat model | one Gaussian | engine scene slot, `checkpoint/SplatPly.h` | +| triangle mesh | vertex, face, component | `meshing::MeshData`, `mesh/MeshImport.cpp` | +| image mask | pixel, shape | `app::FrameMask`, `SegmentPanel` | + +Each already has a reader, a renderer and a writer. What none of them has is a +**mutable document with a history**. That object is the whole of phase 1, and +the four kinds differ only in what an element is. + +## The four objects + +### `EditDoc` — what is open, and what has been done to it + +One per pane. It holds the loaded original, an ordered list of operations, the +current selection, and a dirty flag. The decision that matters: + +> An edit is an entry in a list, not a mutation of the loaded data. + +Replaying the list from the original is how undo works, how "save the edits, +not the result" works, and how the same edits survive the model being retrained +underneath them. A 15M-splat model is about a gigabyte of device memory; a +snapshot per undo step is not affordable, and a list of ops is a few hundred +bytes per step. Where replay is expensive, keep a periodic snapshot in the +on-disk cache — the checkpoint machinery already writes and reads splat PLYs — +and replay from the nearest one. + +### `Selection` — a set, owned by the document and not by any tool + +A bit per element, device-resident for the splat and mesh documents, with a +host mirror pulled only when something needs a count or a histogram. Make it a +`uint8` weight rather than a bit from the start: a soft edge costs the same +memory as a hard one, and the training-region-of-interest feature below wants a +weight anyway. + +Every tool writes into the *same* selection through a combine mode — replace, +add, subtract, intersect — bound to the usual modifiers. That single decision +is what makes "box, minus a brush stroke, intersected with *blue and low +opacity*" work without the box tool and the colour tool knowing about each +other, and it is the reason the histogram selector needs no special case for +"inside this region". + +### `Tool` — a modal state machine over viewport input + +Exactly one active at a time: + +```c++ +struct Tool { + virtual void on_enter(EditCtx&) {} + virtual Consumed on_event(const InputEvent&, EditCtx&) = 0; + virtual void draw_overlay(ImDrawList*, const EditCtx&) {} + virtual const Msg& status_hint() const = 0; // the strip at the bottom +}; +``` + +Camera navigation is itself the default tool. That is what keeps "does this +drag orbit or lasso?" from being a question each feature answers for itself. + +### `Op` — the only thing allowed to change a document + +`apply`, `undo`, a name (a `Msg`, because it appears in the undo menu) and a +serialized form. Nothing else writes to the document. An op script is then a +file, which is what makes the whole thing testable without a window. + +## Making a set + +**A 2D region, extruded.** Box, ellipse, lasso, polygon and brush are one +thing: a screen-space stencil. The tool rasterizes its shape into a bitmask on +the CPU — that part is cheap and different per shape — and one kernel does the +rest, projecting each element with the current view matrix and testing the +bitmask. One kernel, every shape, both backends. + +Two modifiers make it usable rather than a demo: *front-most only*, against the +depth the render already produced, and a depth range taken from two clicks. +Without them, a lasso around a chair also takes the wall behind it, which is +the first thing anyone tries. + +A splat is not a point, so the test needs a policy: by centre, or by any part +of the projected extent. The projection kernel already computes that extent; +offer both and default to the centre. + +**3D primitives.** An oriented box with a gizmo, a sphere, a half-space from a +plane. These are what "crop the scene" actually means, and unlike a screen +region they survive a camera move, which makes them the right thing to *store* +on a document as a reusable clip. + +**Attribute predicates — the histogram selector.** Any per-element scalar +becomes a brushable histogram: opacity, largest and smallest scale, the +anisotropy the `erank` regularizer already computes from the scales, the DC +colour in a chosen space, distance to the nearest training camera, the +accumulated gradient densification already tracks, the observation count of a +sparse point. Two of them at once is a 2D density plot with a rubber-band box. + +Two kernels serve all of it: reduce an attribute into 256 bins over an optional +mask, and threshold it back into the selection. The attributes go in one table +with a name, a getter and a suggested scale, so the panel is generated from the +table rather than written once per attribute. + +The "select the blue-white sky splats tangled into the tree branches" case is +this composed with a region, and it needs nothing new — paint roughly over the +tree with the brush, then *intersect* with a colour and opacity box. That +composition is the feature; neither half is. + +**Connected components.** A grid-backed union-find over element positions. +This is "remove the floaters" and it is also the machinery the sparse-model +split below needs. + +**Grow, shrink, smooth.** A k-NN dilation of the selection. Small, and the +difference between a brush selection that is usable and one that is not. + +## Doing something with it + +Delete. Isolate (keep only). Transform. Recolour, set opacity, clamp scale. +Export the subset. Lock, so densification and later edits leave it alone. + +Two are worth more than the rest: + +**Assign to a group.** A named, saved selection. This is what "segment the +model into components" means in practice, it is what makes a selection +reusable, exportable and re-editable, and it gives the histogram and the region +tools something to write to that outlives the click. + +**Set a training weight.** The region-of-interest ask is not an edit of a +model, it is an input to the *next run*, and it should not be stored in a splat +file. A group exported as an index list with a weight, named by a training +flag, keeps it where it belongs. Note the constraint that comes with it: the +indices only mean anything as long as densification has not renumbered +everything, so the weight has to be carried as a spatial region or re-derived +per run, not as a list of integers that silently rots. + +## Transform, Blender-style + +Two layers, and they are different features: + +- the **gizmo**: axis and plane handles drawn in the viewport; +- the **modal operator**: `G` / `R` / `S`, then `X` / `Y` / `Z` (twice for the + complementary plane), typed numbers, `Shift` for precision, `Ctrl` for snap, + `Enter` or left-click to confirm, `Esc` or right-click to cancel. + +Ship the modal operator first. It is the grammar experienced users actually +want, and it is the *easier* of the two: a small state machine over key events +with a live preview and no hit-testing at all. + +Say the pivot and the axis frame once, in the transform context — median point, +3D cursor, bounding-box centre, individual origins, in global, local or view +axes — and every tool inherits them. + +**Where a transform lives.** `ViewportPanel::set_model_transform` already +places a whole model by moving the *camera* instead of the geometry +(`docs/notes/compare-view.md`), which costs nothing and is the right preview +path. It stops working the moment a transform applies to a subset. So: a +whole-model placement stays camera-side until it is saved, and a subset +transform is an op that rewrites the elements. + +For splats that rewrite is `mean -> sRx + t`, `quat -> Rq`, `log scale -> +log +s` — and the spherical harmonics have to be **rotated** with the model, band by +band, or the view-dependent colour swims as the object turns. That is the one +part of "just rotate it" that is real work rather than three lines, and it is +the same rotation a dataset-level pose normalization would want. + +## The 2D half: masks, the pen tool, intelligent scissors + +`app::FrameMask` is further along than it looks. It already holds an *ordered* +list of keep/remove ellipses and rectangles normalized to the frame, plus an +image stencil intersected with them, and `SegmentPanel` already drags those +shapes over a real decoded frame. Four things are missing: + +- **A path shape.** A closed polygon or Bezier with the same keep/remove + semantics, in the same ordered list: one more `MaskShape::Kind`, one more + case in `parse_mask_shapes`, and a scanline fill in + `rasterize_frame_mask`. +- **Livewire (intelligent scissors).** Dijkstra over a cost image built from + gradient magnitude and direction; the user drops anchors and the path snaps + to the edge between them. It is a couple of hundred lines over an image the + panel has already decoded, it runs on the CPU at preview resolution, and the + cost image is computed once per frame shown. +- **A paint layer.** `FrameMask::image` is already a stencil image, so the + storage exists; what is new is a canvas to paint on and a convention for + where the PNG is written next to the dataset. +- **Refining an AI mask.** The composition rule is already "shapes ∩ image". + Keep the model's output immutable and store the corrections as a separate + layer composed on top; a correction that is destroyed by re-running the model + is a correction nobody will make twice. + +One data-model change is unavoidable: today a stencil belongs to a *camera*, +and hand corrections belong to a *frame*. Both have to exist, keyed the way +`DatasetPrep` already keys cameras. + +## Camera trajectories and video export + +A trajectory is keyframes (pose, field of view, time, and the render options — +a fly-through that changes buffer halfway is a legitimate thing to want), an +interpolation (Catmull-Rom on position, slerp/squad on rotation, with an +optional constant-speed reparameterization so a dense cluster of keyframes does +not crawl), and a render job. + +Almost everything is already here: `NavCamera` for the pose, `RenderWorker` for +the frames, `app/WriterPool.h` for encoding them off the render thread, and an +ffmpeg dependency the app already shells out to. What is missing is UI — a +timeline strip, "key the current view", a curve drawn in the viewport, a scrub +— plus a JSON file stored next to the model so the same move can be re-rendered +after the model is retrained. The turntable and orbit presets are the same +object with the keyframes generated. + +## Splitting and merging a sparse reconstruction + +`src/sfm/map/Merge.h` was written for this: `alignReconstructions`, `mergeInto` +and `MergeSession` are separated, in its own words, *because a GUI will drive +them individually*. So merge is mostly wiring plus a preview of the alignment +before it is committed. + +Split is the new half, and it needs a stated rule rather than an implementation: +an image belongs to a submodel as much as a point does, and a cut leaves tracks +straddling it. Decide once — a track follows the majority of its observations, +ties are dropped — and show the surviving track count *before* the cut is +applied. A destructive operation whose result is a number nobody can predict +needs that number on screen first. + +## Undo, and what it costs + +Start with the cheap trick: **deletes are soft**. A deleted element is marked, +not removed, and compaction happens at save. Undo of a delete is then one bit, +hide and isolate become the same op with a different flag, and the machinery is +the selection bitset that already exists. The cost is that the document carries +its garbage until it is written out, which is the right trade. + +For the ops that cannot be soft, carry the inverse where it is cheap (any +transform) and a compact delta where it is not. Size the delta honestly before +choosing: one splat at SH degree 3 is 59 floats — 236 bytes — so a one-million +splat delta is 236 MB, which belongs in a disk-backed spill, not in RAM behind +a menu the user does not know is holding it. Cap the history by count *and* by +bytes, show what it is holding, and never let it be the reason a session runs +out of memory. + +## Where the code goes + +``` +src/app/gui/edit/ + EditDoc.{h,cpp} the document, the op list, undo/redo + Selection.{h,cpp} the mask, combine modes, named groups + Tool.h the tool interface and the active-tool stack + tools/ SelectBox, SelectLasso, SelectBrush, Transform, Pen, ... + Ops.{h,cpp} delete, transform, recolour, assign to group, ... + Gizmo.{h,cpp} + Attributes.{h,cpp} the per-element scalar table + the brushable histogram + Trajectory.{h,cpp} +src/engine/EngineEdit.cpp engine_select_*, engine_apply_transform, ... +src/shaders/select.slang the stencil test and the attribute reduction +``` + +Rules already in force that this work has to obey, listed because each one is +cheaper to follow than to retrofit: + +- Selection and transform kernels are **Slang, on both backends**. Nothing here + may become CUDA-only (`AGENTS.md`, "the two-backend rule"). +- Every visible string is a `Msg`: a tool name, a status hint, and an op name + as it appears in the undo menu — which means it is a sentence with a `{0}`, + never fragments concatenated. +- A **keyboard shortcut is not interface copy**. `G` / `R` / `S` stay `G` / + `R` / `S` in every language, exactly as `--sh-degree` does. +- Any new setting that gets saved needs its row in the preset field table, or + it will save, load, and quietly run at its default. +- The engine is a process-global singleton and `CompareView` holds one mutex + for every pane. An edit runs under that mutex, between renders, like + everything else that touches the engine. + +## Order of work + +Each phase is shippable on its own and reuses the previous seam rather than +widening it. + +1. **The spine.** `EditDoc`, `Selection`, `Op`, undo, soft delete, one + box-select tool, "delete selection", "save as" — on the splat document in + the viewer screen only. This is already the most-asked-for cleanup + workflow, and everything later is an addition to a working thing. +2. **More ways to select.** Lasso, polygon, brush, invert, grow/shrink, the + depth modifiers. Nothing else changes. +3. **The histogram panel and named groups.** +4. **Transform.** The modal operator, then the gizmo, then SH rotation, then + baking a placement on save. +5. **The mask editor.** Path shape, livewire, paint layer, per-frame + corrections. +6. **Trajectories and video export.** +7. **Sparse split/merge, and region-of-interest weighting for training.** + +## What will bite + +- **The engine singleton.** A scene-slot model is not the training world. Keep + an edit to one from reaching the other, or a run inherits it. +- **Editing then continuing training is not free.** A scene slot carries no + optimizer state, so resuming training on an edited model means resizing the + Adam moments and the densification arrays in the same order as the splats. + `checkpoint/Adapt.cpp` already does host-side layout adaptation on resume; + that is the code to reuse rather than a second one. +- **i18n volume.** An editing UI is a few hundred new messages across thirteen + languages, and the embedded CJK faces are subset to the characters the + catalogs use — a new character with no font regeneration is a hollow box + mid-sentence. +- **ImGui identity.** The viewport is one item and every tool overlay shares + its ID space; push an ID per tool, or two tools' handles collide. +- **Picking.** `RenderWorker` already returns the 3D point under a pixel from + the ray-depth channel it downloads anyway. That is the 3D cursor and + click-to-place; a second picking path would be a second answer to the same + question. +- **Mouse conventions.** The viewport already orbits on the left button, which + collides with Blender's "left confirms, right cancels" in a modal operator. + Settle it once: with no tool active navigation keeps its buttons, and an + active tool owns both buttons for its whole lifetime. +- **Discoverability.** A modal grammar is invisible. A status strip naming the + active tool and its two or three keys is not decoration — for this style of + UI it is the feature. + +## Testing it + +Two levels, and the cheaper one should carry most of the coverage. + +An op list is serializable, so "apply this op script to this model and compare +the result" is a golden-file test that needs no window and no GPU beyond the +one the kernels run on. Every selection producer and every operation is +testable that way. + +The interaction itself — that a drag in the viewport produces the selection the +user meant — needs the real window, and that is what +[gui-automation.md](gui-automation.md) is for: the harness drives the tools +through the same input path a user does and hands back the framebuffer, so a +tool can be exercised and eyeballed without a human at the keyboard. diff --git a/src/app/gui/Automation.cpp b/src/app/gui/Automation.cpp new file mode 100644 index 00000000..9447d1d5 --- /dev/null +++ b/src/app/gui/Automation.cpp @@ -0,0 +1,804 @@ +// Automation.cpp -- see Automation.h. + +#include "app/gui/Automation.h" + +#include "app/AppPaths.h" +#include "app/webviewer/HttpServer.h" +#include "core/Env.h" +#include "app/gui/GlLoader.h" +#include "external/stb_image_write.h" + +#include "imgui.h" +#include "imgui_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// =========================================================================== +// State +// =========================================================================== + +// One widget as the last completed frame saw it. `key` is what follows "###" +// in the label -- the i18n message name (src/app/gui/Ui.h), which is the same +// in every language and is what a script addresses. +struct Item { + unsigned id = 0; + std::string key; + std::string label; + std::string window; + float x0 = 0, y0 = 0, x1 = 0, y1 = 0; + int flags = 0; +}; + +// One frame's worth of injected input; several make a gesture. Exactly one +// is applied per frame, which is what keeps a move and the click after it in +// separate frames -- the hover test between them is what makes the click land. +struct Step { + enum class Kind { None, MousePos, MouseButton, Wheel, Key, Text, Focus }; + Kind kind = Kind::None; + float x = 0, y = 0; + int button = 0; + bool down = false; + std::vector keys; // ImGuiKey + std::string text; +}; + +struct State { + bool armed = false; + HttpServer http; + std::string token; + std::function state_source; + + std::mutex mu; + std::condition_variable cv; + + std::vector building; // GUI thread only, the frame in flight + std::vector published; // guarded by mu + uint64_t frame_no = 0; // guarded by mu + float display_w = 0, display_h = 0, fb_scale = 1; + int fb_w = 0, fb_h = 0; + + std::deque queue; // guarded by mu + uint64_t queued = 0, applied = 0; + + bool want_shot = false; // guarded by mu + bool shot_ready = false; + int shot_w = 0; // 0 = the framebuffer's own width + bool shot_jpeg = false; + int shot_quality = 85; + std::vector shot; +}; + +State& st() { + static State s; + return s; +} + +constexpr size_t kMaxItems = 8192; + +// =========================================================================== +// JSON +// =========================================================================== + +void json_str(std::string& out, const std::string& s) { + out += '"'; + for (unsigned char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += (char)c; + } + } + } + out += '"'; +} + +std::string json_num(double v) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "%.2f", v); + return buf; +} + +HttpResponse ok_json(const std::string& body) { return HttpResponse::json(body); } + +HttpResponse err_json(int status, const std::string& what) { + std::string body = "{\"ok\":false,\"error\":"; + json_str(body, what); + body += "}"; + return HttpResponse::text(status, body, "application/json"); +} + +// =========================================================================== +// Keys +// =========================================================================== + +struct KeyName { const char* name; ImGuiKey key; }; + +const KeyName kKeys[] = { + {"tab", ImGuiKey_Tab}, {"left", ImGuiKey_LeftArrow}, + {"right", ImGuiKey_RightArrow}, {"up", ImGuiKey_UpArrow}, + {"down", ImGuiKey_DownArrow}, {"pageup", ImGuiKey_PageUp}, + {"pagedown", ImGuiKey_PageDown}, {"home", ImGuiKey_Home}, + {"end", ImGuiKey_End}, {"insert", ImGuiKey_Insert}, + {"delete", ImGuiKey_Delete}, {"backspace", ImGuiKey_Backspace}, + {"space", ImGuiKey_Space}, {"enter", ImGuiKey_Enter}, + {"escape", ImGuiKey_Escape}, {"esc", ImGuiKey_Escape}, + {"ctrl", ImGuiKey_LeftCtrl}, {"shift", ImGuiKey_LeftShift}, + {"alt", ImGuiKey_LeftAlt}, {"super", ImGuiKey_LeftSuper}, + {"minus", ImGuiKey_Minus}, {"equal", ImGuiKey_Equal}, + {"comma", ImGuiKey_Comma}, {"period", ImGuiKey_Period}, + {"slash", ImGuiKey_Slash}, +}; + +std::string lower(std::string s) { + for (char& c : s) c = (char)std::tolower((unsigned char)c); + return s; +} + +// "Ctrl+Shift+A" -> the key events to hold, innermost last. Empty on a name +// this table does not know. +bool parse_chord(const std::string& spec, std::vector& out) { + size_t pos = 0; + while (pos <= spec.size()) { + size_t plus = spec.find('+', pos); + std::string part = lower(spec.substr(pos, plus == std::string::npos + ? std::string::npos + : plus - pos)); + if (part.empty()) return false; + ImGuiKey k = ImGuiKey_None; + for (const KeyName& kn : kKeys) + if (part == kn.name) { k = kn.key; break; } + if (k == ImGuiKey_None && part.size() == 1) { + char c = part[0]; + if (c >= 'a' && c <= 'z') k = (ImGuiKey)(ImGuiKey_A + (c - 'a')); + else if (c >= '0' && c <= '9') k = (ImGuiKey)(ImGuiKey_0 + (c - '0')); + } + if (k == ImGuiKey_None && part.size() >= 2 && part[0] == 'f') { + int n = std::atoi(part.c_str() + 1); + if (n >= 1 && n <= 12) k = (ImGuiKey)(ImGuiKey_F1 + (n - 1)); + } + if (k == ImGuiKey_None) return false; + out.push_back((int)k); + if (plus == std::string::npos) break; + pos = plus + 1; + } + return !out.empty(); +} + +// The modifier ImGui derives from a physical key, so a chord reaches a +// shortcut test as well as a key-down test. +ImGuiKey mod_of(int key) { + switch ((ImGuiKey)key) { + case ImGuiKey_LeftCtrl: case ImGuiKey_RightCtrl: return ImGuiMod_Ctrl; + case ImGuiKey_LeftShift: case ImGuiKey_RightShift: return ImGuiMod_Shift; + case ImGuiKey_LeftAlt: case ImGuiKey_RightAlt: return ImGuiMod_Alt; + case ImGuiKey_LeftSuper: case ImGuiKey_RightSuper: return ImGuiMod_Super; + default: return ImGuiKey_None; + } +} + +// =========================================================================== +// Queue +// =========================================================================== + +uint64_t enqueue(const std::vector& steps) { + State& s = st(); + std::lock_guard lk(s.mu); + for (const Step& step : steps) s.queue.push_back(step); + s.queued += steps.size(); + return s.queued; +} + +// Blocks until every step queued up to `mark` has been applied and one more +// frame has been drawn with the result. False on timeout. +bool wait_applied(uint64_t mark, double timeout_s) { + State& s = st(); + std::unique_lock lk(s.mu); + uint64_t want_frame = 0; + if (!s.cv.wait_for(lk, std::chrono::duration(timeout_s), + [&] { return s.applied >= mark; })) + return false; + want_frame = s.frame_no + 2; + return s.cv.wait_for(lk, std::chrono::duration(timeout_s), + [&] { return s.frame_no >= want_frame; }); +} + +// A move, then the press and the release, each in its own frame. `settle` +// frames of nothing follow, which is what lets a popup open before the next +// command looks for what is in it. +void push_click(std::vector& out, float x, float y, int button, + bool dbl, int settle) { + Step move; + move.kind = Step::Kind::MousePos; + move.x = x; move.y = y; + out.push_back(move); + out.push_back(move); + for (int i = 0; i < (dbl ? 2 : 1); i++) { + Step b; + b.kind = Step::Kind::MouseButton; + b.button = button; + b.down = true; + out.push_back(b); + b.down = false; + out.push_back(b); + } + for (int i = 0; i < settle; i++) out.push_back(Step{}); +} + +// =========================================================================== +// Item lookup +// =========================================================================== + +bool find_item(const std::string& key, int index, Item& out, std::string& err) { + State& s = st(); + std::lock_guard lk(s.mu); + int seen = 0; + for (const Item& it : s.published) { + if (it.key != key) continue; + if (seen++ != index) continue; + if (it.x1 <= it.x0 || it.y1 <= it.y0) { + err = "item has an empty rect: " + key; + return false; + } + out = it; + return true; + } + err = seen == 0 ? "no item on screen with id: " + key + : "only " + std::to_string(seen) + " item(s) with id: " + key; + return false; +} + +} // namespace + +// =========================================================================== +// ImGui item hooks +// +// Declared by imgui_internal.h under IMGUI_ENABLE_TEST_ENGINE. Nothing runs +// until ctx->TestEngineHookItems is set, which only arm() does. +// =========================================================================== + +void ImGuiTestEngineHook_ItemAdd(ImGuiContext* ctx, ImGuiID id, const ImRect& bb, + const ImGuiLastItemData* item_data) { + (void)item_data; + State& s = st(); + if (s.building.size() >= kMaxItems) return; + Item it; + it.id = (unsigned)id; + it.x0 = bb.Min.x; it.y0 = bb.Min.y; + it.x1 = bb.Max.x; it.y1 = bb.Max.y; + if (ctx->CurrentWindow) it.window = ctx->CurrentWindow->Name; + s.building.push_back(std::move(it)); +} + +void ImGuiTestEngineHook_ItemInfo(ImGuiContext* ctx, ImGuiID id, const char* label, + ImGuiItemStatusFlags flags) { + (void)ctx; + State& s = st(); + for (size_t i = s.building.size(); i-- > 0; ) { + if (s.building[i].id != (unsigned)id) continue; + Item& it = s.building[i]; + it.flags = (int)flags; + if (!label) return; + const char* hash = std::strstr(label, "###"); + if (hash) { + it.label.assign(label, hash - label); + it.key = hash + 3; + } else { + const char* hide = std::strstr(label, "##"); + it.label.assign(label, hide ? hide - label : std::strlen(label)); + it.key = label; + } + return; + } +} + +void ImGuiTestEngineHook_Log(ImGuiContext* ctx, const char* fmt, ...) { + (void)ctx; (void)fmt; +} + +const char* ImGuiTestEngine_FindItemDebugLabel(ImGuiContext* ctx, ImGuiID id) { + (void)ctx; + State& s = st(); + for (const Item& it : s.building) + if (it.id == (unsigned)id && !it.label.empty()) return it.label.c_str(); + return nullptr; +} + +// =========================================================================== +// Endpoints +// =========================================================================== + +namespace { + +std::string state_json() { + State& s = st(); + std::string out = "{\"ok\":true"; + { + std::lock_guard lk(s.mu); + out += ",\"frame\":" + std::to_string(s.frame_no); + out += ",\"items\":" + std::to_string(s.published.size()); + out += ",\"queued\":" + std::to_string(s.queued - s.applied); + out += ",\"display\":[" + json_num(s.display_w) + "," + + json_num(s.display_h) + "]"; + out += ",\"framebuffer\":[" + std::to_string(s.fb_w) + "," + + std::to_string(s.fb_h) + "]"; + out += ",\"fb_scale\":" + json_num(s.fb_scale); + } + if (s.state_source) { + std::string extra = s.state_source(); + if (!extra.empty()) out += "," + extra; + } + out += "}"; + return out; +} + +HttpResponse handle_tree(const HttpRequest& r) { + const std::string q = lower(r.get("q")); + const std::string window = r.get("window"); + const bool named_only = r.get_bool("named", true); + State& s = st(); + std::string out = "{\"ok\":true,\"items\":["; + bool first = true; + std::lock_guard lk(s.mu); + for (const Item& it : s.published) { + if (named_only && it.key.empty()) continue; + if (!window.empty() && it.window != window) continue; + if (!q.empty() && lower(it.key).find(q) == std::string::npos && + lower(it.label).find(q) == std::string::npos) + continue; + if (!first) out += ","; + first = false; + out += "{\"id\":"; + json_str(out, it.key); + out += ",\"label\":"; + json_str(out, it.label); + out += ",\"window\":"; + json_str(out, it.window); + out += ",\"rect\":[" + json_num(it.x0) + "," + json_num(it.y0) + "," + + json_num(it.x1) + "," + json_num(it.y1) + "]"; + out += ",\"flags\":" + std::to_string(it.flags) + "}"; + } + out += "]}"; + return ok_json(out); +} + +// Where a command acts: the centre of the item named by `id`, or the `at=x,y` +// the caller gave instead. +bool resolve_point(const HttpRequest& r, float& x, float& y, std::string& err) { + const std::string at = r.get("at"); + if (!at.empty()) { + if (std::sscanf(at.c_str(), "%f,%f", &x, &y) != 2) { + err = "at= wants x,y"; + return false; + } + return true; + } + const std::string id = r.get("id"); + if (id.empty()) { + err = "give id= or at="; + return false; + } + Item it; + if (!find_item(id, r.get_int("index", 0), it, err)) return false; + x = 0.5f * (it.x0 + it.x1); + y = 0.5f * (it.y0 + it.y1); + return true; +} + +HttpResponse finish(const HttpRequest& r, uint64_t mark) { + if (r.get_bool("nowait", false)) return ok_json("{\"ok\":true}"); + if (!wait_applied(mark, r.get_double("timeout", 15.0))) + return err_json(504, "timed out waiting for the frame loop"); + return ok_json(state_json()); +} + +HttpResponse handle_click(const HttpRequest& r) { + float x = 0, y = 0; + std::string err; + if (!resolve_point(r, x, y, err)) return err_json(404, err); + std::vector steps; + push_click(steps, x, y, r.get_int("button", 0), r.get_bool("double", false), + r.get_int("settle", 2)); + return finish(r, enqueue(steps)); +} + +HttpResponse handle_move(const HttpRequest& r) { + float x = 0, y = 0; + std::string err; + if (!resolve_point(r, x, y, err)) return err_json(404, err); + Step m; + m.kind = Step::Kind::MousePos; + m.x = x; m.y = y; + return finish(r, enqueue({m, Step{}})); +} + +HttpResponse handle_drag(const HttpRequest& r) { + float x0 = 0, y0 = 0, x1 = 0, y1 = 0; + if (std::sscanf(r.get("from").c_str(), "%f,%f", &x0, &y0) != 2 || + std::sscanf(r.get("to").c_str(), "%f,%f", &x1, &y1) != 2) + return err_json(400, "drag wants from=x,y and to=x,y"); + const int n = std::max(1, std::min(256, r.get_int("steps", 8))); + const int button = r.get_int("button", 0); + std::vector steps; + Step m; + m.kind = Step::Kind::MousePos; + m.x = x0; m.y = y0; + steps.push_back(m); + steps.push_back(m); + Step b; + b.kind = Step::Kind::MouseButton; + b.button = button; + b.down = true; + steps.push_back(b); + for (int i = 1; i <= n; i++) { + Step p; + p.kind = Step::Kind::MousePos; + p.x = x0 + (x1 - x0) * (float)i / (float)n; + p.y = y0 + (y1 - y0) * (float)i / (float)n; + steps.push_back(p); + } + b.down = false; + steps.push_back(b); + steps.push_back(Step{}); + return finish(r, enqueue(steps)); +} + +HttpResponse handle_scroll(const HttpRequest& r) { + float x = 0, y = 0; + std::string err; + if (!resolve_point(r, x, y, err)) return err_json(404, err); + Step m; + m.kind = Step::Kind::MousePos; + m.x = x; m.y = y; + Step w; + w.kind = Step::Kind::Wheel; + w.x = (float)r.get_double("dx", 0.0); + w.y = (float)r.get_double("dy", -1.0); + return finish(r, enqueue({m, m, w, Step{}})); +} + +HttpResponse handle_key(const HttpRequest& r) { + const std::string spec = r.get("keys"); + if (spec.empty()) return err_json(400, "key wants keys=Ctrl+S"); + std::vector chord; + if (!parse_chord(spec, chord)) return err_json(400, "unknown key: " + spec); + Step down, up; + down.kind = up.kind = Step::Kind::Key; + down.keys = chord; + down.down = true; + up.keys = chord; + up.down = false; + return finish(r, enqueue({down, up, Step{}, Step{}})); +} + +HttpResponse handle_text(const HttpRequest& r) { + float x = 0, y = 0; + std::string err; + if (!resolve_point(r, x, y, err)) return err_json(404, err); + std::vector steps; + push_click(steps, x, y, 0, false, 1); + std::vector select_all{(int)ImGuiKey_LeftCtrl, (int)ImGuiKey_A}; + Step down, up; + down.kind = up.kind = Step::Kind::Key; + down.keys = select_all; + down.down = true; + up.keys = select_all; + up.down = false; + steps.push_back(down); + steps.push_back(up); + Step t; + t.kind = Step::Kind::Text; + t.text = r.get("value"); + steps.push_back(t); + steps.push_back(Step{}); + if (r.get_bool("enter", true)) { + std::vector ret{(int)ImGuiKey_Enter}; + Step ed, eu; + ed.kind = eu.kind = Step::Kind::Key; + ed.keys = ret; + ed.down = true; + eu.keys = ret; + eu.down = false; + steps.push_back(ed); + steps.push_back(eu); + } + steps.push_back(Step{}); + return finish(r, enqueue(steps)); +} + +HttpResponse handle_wait(const HttpRequest& r) { + const int frames = std::max(0, std::min(600, r.get_int("frames", 2))); + std::vector steps((size_t)frames + 1, Step{}); + return finish(r, enqueue(steps)); +} + +// Area average, which is what a screenshot wants: a downscale that samples +// would drop every one-pixel line the interface is drawn with. +std::vector box_resize(const std::vector& src, int sw, int sh, + int dw, int dh) { + std::vector dst((size_t)dw * dh * 4); + for (int y = 0; y < dh; y++) { + const int y0 = (int)((int64_t)y * sh / dh); + const int y1 = std::max(y0 + 1, (int)((int64_t)(y + 1) * sh / dh)); + for (int x = 0; x < dw; x++) { + const int x0 = (int)((int64_t)x * sw / dw); + const int x1 = std::max(x0 + 1, (int)((int64_t)(x + 1) * sw / dw)); + int acc[4] = {0, 0, 0, 0}; + for (int sy = y0; sy < y1; sy++) + for (int sx = x0; sx < x1; sx++) + for (int c = 0; c < 4; c++) + acc[c] += src[((size_t)sy * sw + sx) * 4 + c]; + const int n = (y1 - y0) * (x1 - x0); + for (int c = 0; c < 4; c++) + dst[((size_t)y * dw + x) * 4 + c] = (uint8_t)(acc[c] / n); + } + } + return dst; +} + +void png_write_cb(void* ctx, void* data, int size) { + auto* out = (std::vector*)ctx; + out->insert(out->end(), (uint8_t*)data, (uint8_t*)data + size); +} + +HttpResponse handle_screenshot(const HttpRequest& r) { + State& s = st(); + const bool jpeg = r.get("format", "png") == "jpg" || + r.get("format", "png") == "jpeg"; + std::vector png; + { + std::unique_lock lk(s.mu); + s.shot.clear(); + s.shot_ready = false; + s.shot_w = std::max(0, r.get_int("width", 0)); + s.shot_jpeg = jpeg; + s.shot_quality = std::max(1, std::min(100, r.get_int("quality", 85))); + s.want_shot = true; + if (!s.cv.wait_for(lk, std::chrono::duration( + r.get_double("timeout", 10.0)), + [&] { return s.shot_ready; })) { + s.want_shot = false; + return err_json(504, "the frame loop did not deliver a frame"); + } + png.swap(s.shot); + } + const std::string path = r.get("path"); + if (!path.empty()) { + std::ofstream f(path, std::ios::binary); + if (!f) return err_json(500, "cannot write " + path); + f.write((const char*)png.data(), (std::streamsize)png.size()); + std::string body = "{\"ok\":true,\"bytes\":" + + std::to_string(png.size()) + ",\"path\":"; + json_str(body, path); + body += "}"; + return ok_json(body); + } + HttpResponse out; + out.content_type = jpeg ? "image/jpeg" : "image/png"; + out.body = std::move(png); + return out; +} + +// Every route goes through this: the server is on the loopback interface, but +// a page in a browser can still reach it, and these are GETs with effects. +HttpServer::Handler guard(HttpServer::Handler h) { + return [h](const HttpRequest& r) -> HttpResponse { + if (!st().token.empty() && r.get("token") != st().token) + return err_json(403, "bad or missing token="); + return h(r); + }; +} + +std::string make_token() { + std::random_device rd; + std::uniform_int_distribution d(0, 15); + std::string t; + for (int i = 0; i < 32; i++) t += "0123456789abcdef"[d(rd)]; + return t; +} + +} // namespace + +// =========================================================================== +// Frame loop +// =========================================================================== + +namespace gui { +namespace automation { + +bool armed() { return st().armed; } + +void set_state_source(std::function f) { + st().state_source = std::move(f); +} + +void arm() { + if (!spirula::env_on("GUI_AUTOMATION")) return; + State& s = st(); + + ImGuiContext* ctx = ImGui::GetCurrentContext(); + if (!ctx) return; + ctx->TestEngineHookItems = true; + // A driven window is rarely the focused one, and ImGui clears the input + // state it was handed when it believes focus was lost. + ImGui::GetIO().ConfigDebugIgnoreFocusLoss = true; + + if (const char* t = spirula::env("GUI_AUTOMATION_TOKEN")) s.token = t; + else s.token = make_token(); + + const char* port_env = spirula::env("GUI_AUTOMATION_PORT"); + const int port = port_env ? std::atoi(port_env) : 7777; + + s.http.route("/ui/state", guard([](const HttpRequest&) { + return ok_json(state_json()); + })); + s.http.route("/ui/tree", guard(handle_tree)); + s.http.route("/ui/click", guard(handle_click)); + s.http.route("/ui/move", guard(handle_move)); + s.http.route("/ui/drag", guard(handle_drag)); + s.http.route("/ui/scroll", guard(handle_scroll)); + s.http.route("/ui/key", guard(handle_key)); + s.http.route("/ui/text", guard(handle_text)); + s.http.route("/ui/wait", guard(handle_wait)); + s.http.route("/ui/screenshot", guard(handle_screenshot)); + + try { + s.http.start("127.0.0.1", port); + } catch (const std::exception& e) { + ctx->TestEngineHookItems = false; + std::fprintf(stderr, "[automation] not listening: %s\n", e.what()); + return; + } + s.armed = true; + + const std::string file = app::config_dir() + "/automation.json"; + std::ofstream f(file); + f << "{\"host\":\"127.0.0.1\",\"port\":" << port << ",\"token\":\"" + << s.token << "\"}\n"; + std::fprintf(stderr, "[automation] http://127.0.0.1:%d token in %s\n", + port, file.c_str()); +} + +bool begin_frame() { + State& s = st(); + if (!s.armed) return false; + s.building.clear(); + + Step step; + bool popped = false; + size_t left = 0; + { + std::lock_guard lk(s.mu); + if (!s.queue.empty()) { + step = s.queue.front(); + s.queue.pop_front(); + popped = true; + } + left = s.queue.size(); + } + + ImGuiIO& io = ImGui::GetIO(); + switch (step.kind) { + case Step::Kind::None: break; + case Step::Kind::MousePos: io.AddMousePosEvent(step.x, step.y); break; + case Step::Kind::MouseButton: + io.AddMouseButtonEvent(step.button, step.down); + break; + case Step::Kind::Wheel: io.AddMouseWheelEvent(step.x, step.y); break; + case Step::Kind::Key: { + // Held keys go down outermost-first and come up in the same order, + // so a modifier is already down when the key it modifies arrives. + for (int k : step.keys) { + if (ImGuiKey m = mod_of(k); m != ImGuiKey_None) + io.AddKeyEvent(m, step.down); + io.AddKeyEvent((ImGuiKey)k, step.down); + } + break; + } + case Step::Kind::Text: + io.AddInputCharactersUTF8(step.text.c_str()); + break; + case Step::Kind::Focus: io.AddFocusEvent(true); break; + } + + if (popped) { + std::lock_guard lk(s.mu); + s.applied++; + } + s.cv.notify_all(); + return popped || left > 0; +} + +void end_frame(int fb_w, int fb_h) { + State& s = st(); + if (!s.armed) return; + + bool shoot = false; + int want_w = 0, quality = 85; + bool jpeg = false; + { + std::lock_guard lk(s.mu); + s.published.swap(s.building); + s.frame_no++; + const ImGuiIO& io = ImGui::GetIO(); + s.display_w = io.DisplaySize.x; + s.display_h = io.DisplaySize.y; + s.fb_w = fb_w; + s.fb_h = fb_h; + s.fb_scale = io.DisplaySize.x > 0 ? (float)fb_w / io.DisplaySize.x : 1.0f; + shoot = s.want_shot; + want_w = s.shot_w; + jpeg = s.shot_jpeg; + quality = s.shot_quality; + } + s.building.clear(); + if (!shoot || fb_w <= 0 || fb_h <= 0) { + s.cv.notify_all(); + return; + } + + std::vector rgba((size_t)fb_w * fb_h * 4); + glPixelStorei(GL_PACK_ALIGNMENT, 1); + glReadPixels(0, 0, fb_w, fb_h, GL_RGBA, GL_UNSIGNED_BYTE, rgba.data()); + // GL reads bottom-up; every PNG reader expects the other one. + const size_t row = (size_t)fb_w * 4; + std::vector flip((size_t)fb_w * fb_h * 4); + for (int y = 0; y < fb_h; y++) + std::memcpy(flip.data() + (size_t)y * row, + rgba.data() + (size_t)(fb_h - 1 - y) * row, row); + + int out_w = fb_w, out_h = fb_h; + if (want_w > 0 && want_w < fb_w) { + out_w = want_w; + out_h = std::max(1, (int)((int64_t)fb_h * want_w / fb_w)); + flip = box_resize(flip, fb_w, fb_h, out_w, out_h); + } + std::vector png; + if (jpeg) + stbi_write_jpg_to_func(png_write_cb, &png, out_w, out_h, 4, + flip.data(), quality); + else + stbi_write_png_to_func(png_write_cb, &png, out_w, out_h, 4, + flip.data(), out_w * 4); + { + std::lock_guard lk(s.mu); + s.shot = std::move(png); + s.shot_ready = true; + s.want_shot = false; + } + s.cv.notify_all(); +} + +void shutdown() { + State& s = st(); + if (!s.armed) return; + s.http.stop(); + s.armed = false; + s.cv.notify_all(); +} + +} // namespace automation +} // namespace gui diff --git a/src/app/gui/Automation.h b/src/app/gui/Automation.h new file mode 100644 index 00000000..ff844520 --- /dev/null +++ b/src/app/gui/Automation.h @@ -0,0 +1,40 @@ +#pragma once + +// Driving this GUI from outside it. +// +// A loopback HTTP surface that lists the widgets currently on screen, injects +// mouse/keyboard input at them, and hands back the framebuffer as a PNG. Off +// unless SS_GUI_AUTOMATION is set, and the imgui item hooks it needs stay +// switched off with it, so a normal run pays one never-taken branch per item. +// +// Design, endpoints and the client: docs/notes/gui-automation.md. + +#include +#include + +namespace gui { +namespace automation { + +// Reads SS_GUI_AUTOMATION and binds the server. Call after +// ImGui::CreateContext() and before the first frame; a bind failure is +// reported on stderr and leaves automation off. +void arm(); +bool armed(); + +// What /ui/state reports about the application on top of what ImGui knows: +// a JSON object body without the braces, or "" for nothing. +void set_state_source(std::function f); + +// Between glfwPollEvents() and ImGui::NewFrame(): applies one step of the +// queued input. True while anything is still queued, which is what keeps the +// frame loop at its busy rate for the length of a script. +bool begin_frame(); + +// After the draw data has been rendered, with the GL context current: +// publishes the frame's item table and fills a pending screenshot request. +void end_frame(int fb_w, int fb_h); + +void shutdown(); + +} // namespace automation +} // namespace gui diff --git a/src/app/gui/GuiApp.cpp b/src/app/gui/GuiApp.cpp index 410dc697..f1912c12 100644 --- a/src/app/gui/GuiApp.cpp +++ b/src/app/gui/GuiApp.cpp @@ -2354,6 +2354,36 @@ void GuiApp::handle_dialog_result(const std::vector& paths) { // Frame // =========================================================================== +std::string GuiApp::state_json() { + auto quoted = [](const std::string& s) { + std::string q = "\""; + for (char c : s) { + if ((unsigned char)c < 0x20) continue; + if (c == '"' || c == '\\') q += '\\'; + q += c; + } + return q + "\""; + }; + // Index order is the declaration order of Screen and TrainRunner::Phase. + static const char* kScreens[] = {"home", "new_dataset", "train", "viewer", + "batch", "mesh"}; + static const char* kPhases[] = {"idle", "loading", "ready", "load_error", + "preparing", "training", "done", + "train_error"}; + std::string out = "\"screen\":\""; + out += kScreens[(int)_screen]; + out += "\",\"train_phase\":\""; + out += kPhases[(int)_runner.phase()]; + out += "\",\"step\":" + std::to_string(_runner.latest_progress().step); + out += ",\"busy\":"; + out += native_work_busy() ? "true" : "false"; + out += ",\"dialog_open\":"; + out += _dialog.is_open() ? "true" : "false"; + out += ",\"models_open\":" + std::to_string(_compare.count()); + out += ",\"dataset\":" + quoted(_cfg.data); + return out; +} + void GuiApp::frame() { // Before the first widget: a style swap halfway through a frame would // measure half the window against one scale and half against the other. diff --git a/src/app/gui/GuiApp.h b/src/app/gui/GuiApp.h index 6d5e9382..bacff842 100644 --- a/src/app/gui/GuiApp.h +++ b/src/app/gui/GuiApp.h @@ -72,6 +72,11 @@ public: // Draw one frame (between ImGui::NewFrame and ImGui::Render). void frame(); + // What a script needs to know that is not on screen as a widget: the + // screen, what is running, what is open. A JSON object body without the + // braces, for gui::automation::set_state_source. + std::string state_json(); + // Window close button pressed; may open a confirmation dialog instead // of quitting when training is in flight. void request_close(); diff --git a/src/app/gui/GuiMain.cpp b/src/app/gui/GuiMain.cpp index 73dfcfd0..bd421541 100644 --- a/src/app/gui/GuiMain.cpp +++ b/src/app/gui/GuiMain.cpp @@ -5,7 +5,9 @@ #include "app/Tools.h" #include "i18n/catalog/Log.h" #include "app/AppPaths.h" +#include "core/Env.h" #include "app/CrashLog.h" +#include "app/gui/Automation.h" #include "app/gui/Fonts.h" #include "app/gui/GuiApp.h" #include "app/gui/Layout.h" @@ -179,6 +181,11 @@ int spirula_gui_main(int argc, char** argv) { if (mw > 0) win_w = std::min(win_w, mw); if (mh > 0) win_h = std::min(win_h, mh); } + // A driven window still needs a real GL context, but nothing needs it on + // screen -- and one that is not on screen cannot be clicked on by accident. + if (spirula::env_on("GUI_OFFSCREEN")) + glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE); + // The title is set from the catalog below, once GuiApp has settled the // language; this is only what the window is born with. GLFWwindow* window = glfwCreateWindow(win_w, win_h, "Spirula Studio", @@ -209,9 +216,11 @@ int spirula_gui_main(int argc, char** argv) { ImGui_ImplGlfw_InitForOpenGL(window, true); ImGui_ImplOpenGL3_Init(glsl_version); + gui::automation::arm(); { gui::GuiApp app; + gui::automation::set_state_source([&app] { return app.state_json(); }); app.set_dpi_scale(layout_dpi(window)); // The atlas has to exist before the first frame, and the language it // depends on is only settled once GuiApp has read its settings file. @@ -274,9 +283,12 @@ int spirula_gui_main(int argc, char** argv) { ImGui_ImplOpenGL3_NewFrame(); ImGui_ImplGlfw_NewFrame(); + // After the backend's own mouse update, so an injected position + // is the later event and wins for this frame. + const bool scripted = gui::automation::begin_frame(); ImGui::NewFrame(); app.frame(); - const bool busy = ui_busy(); + const bool busy = ui_busy() || scripted; ImGui::Render(); const double now = glfwGetTime(); @@ -293,6 +305,7 @@ int spirula_gui_main(int argc, char** argv) { glClearColor(0.07f, 0.07f, 0.08f, 1.0f); glClear(GL_COLOR_BUFFER_BIT); ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData()); + gui::automation::end_frame(w, h); // reads the back buffer glfwSwapBuffers(window); } // Joins worker threads (finishing a final checkpoint save if a stop @@ -301,6 +314,7 @@ int spirula_gui_main(int argc, char** argv) { app.shutdown(); } + gui::automation::shutdown(); ImGui_ImplOpenGL3_Shutdown(); ImGui_ImplGlfw_Shutdown(); ImGui::DestroyContext(); diff --git a/tools/gui_mcp.py b/tools/gui_mcp.py new file mode 100755 index 00000000..ff47dbb0 --- /dev/null +++ b/tools/gui_mcp.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""MCP stdio server over the GUI control surface. + +The same endpoints tools/guictl.py calls, exposed as MCP tools so a coding +agent can list the widgets on screen, click them, and get the framebuffer back +as an image without shelling out and reading files. + +Standard library only, on purpose: this repo takes no Python dependencies, and +MCP over stdio is newline-delimited JSON-RPC 2.0. + +Register it for Claude Code with .mcp.json at the repo root; anything else +that speaks MCP wants `python3 tools/gui_mcp.py` as the stdio command. +""" + +import base64 +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import guictl # noqa: E402 + +PROTOCOL = "2025-06-18" + +TOOLS = [ + { + "name": "gui_launch", + "description": "Start the Spirula Studio GUI with the automation " + "surface armed. Returns once it answers.", + "inputSchema": { + "type": "object", + "properties": { + "offscreen": {"type": "boolean", + "description": "Create the window invisible."}, + "exe": {"type": "string", + "description": "Path to the spirula binary; defaults " + "to build_cuda/spirula."}, + "args": {"type": "array", "items": {"type": "string"}, + "description": "Files to open, as a drop would."}, + }, + }, + }, + { + "name": "gui_state", + "description": "Screen, training phase, dialog and queue state, " + "display and framebuffer sizes.", + "inputSchema": {"type": "object", "properties": {}}, + }, + { + "name": "gui_tree", + "description": "The widgets on screen: id (the i18n message name, " + "stable across languages), visible label, rect, window.", + "inputSchema": { + "type": "object", + "properties": { + "q": {"type": "string", + "description": "Substring of the id or the label."}, + "window": {"type": "string"}, + "all": {"type": "boolean", + "description": "Include items with no id of their own."}, + }, + }, + }, + { + "name": "gui_click", + "description": "Click a widget by id, or a point with at=\"x,y\".", + "inputSchema": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "at": {"type": "string", "description": "x,y"}, + "button": {"type": "integer", + "description": "0 left, 1 right, 2 middle."}, + "double": {"type": "boolean"}, + "index": {"type": "integer", + "description": "Which of several items sharing an id."}, + }, + }, + }, + { + "name": "gui_move", + "description": "Move the pointer onto a widget or point (hover, " + "tooltips).", + "inputSchema": { + "type": "object", + "properties": {"id": {"type": "string"}, "at": {"type": "string"}, + "index": {"type": "integer"}}, + }, + }, + { + "name": "gui_drag", + "description": "Press, move and release: viewport orbit/pan, sliders, " + "and the selection gestures.", + "inputSchema": { + "type": "object", + "properties": { + "from": {"type": "string", "description": "x,y"}, + "to": {"type": "string", "description": "x,y"}, + "button": {"type": "integer"}, + "steps": {"type": "integer", + "description": "Intermediate positions; more for a " + "path a tool has to follow."}, + }, + "required": ["from", "to"], + }, + }, + { + "name": "gui_scroll", + "description": "Wheel over a widget or point (zoom, scrolling panels).", + "inputSchema": { + "type": "object", + "properties": {"id": {"type": "string"}, "at": {"type": "string"}, + "dy": {"type": "number"}}, + }, + }, + { + "name": "gui_key", + "description": "A key chord, e.g. \"Escape\", \"Ctrl+S\", " + "\"Ctrl+Shift+A\".", + "inputSchema": { + "type": "object", + "properties": {"keys": {"type": "string"}}, + "required": ["keys"], + }, + }, + { + "name": "gui_text", + "description": "Focus a text field, select everything in it, and type " + "a replacement.", + "inputSchema": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "at": {"type": "string"}, + "value": {"type": "string"}, + "enter": {"type": "boolean", + "description": "Press Enter afterwards (default true)."}, + }, + "required": ["value"], + }, + }, + { + "name": "gui_wait", + "description": "Let N frames pass, for work that finishes on the GUI " + "thread.", + "inputSchema": { + "type": "object", + "properties": {"frames": {"type": "integer"}}, + }, + }, + { + "name": "gui_screenshot", + "description": "The current framebuffer, as an image.", + "inputSchema": { + "type": "object", + "properties": { + "width": {"type": "integer", + "description": "Downscale to this width (default 1280)."}, + "format": {"type": "string", "enum": ["jpg", "png"], + "description": "Default jpg; png for exact pixels."}, + "quality": {"type": "integer", "description": "jpg, 1-100."}, + "path": {"type": "string", + "description": "Also write the image here."}, + }, + }, + }, +] + + +def point_args(a): + out = {} + if a.get("at"): + out["at"] = a["at"] + elif a.get("id"): + out["id"] = a["id"] + out["index"] = a.get("index", 0) + return out + + +def run_tool(name, a): + """(content list, is_error).""" + if name == "gui_launch": + class Args: + pass + args = Args() + args.exe = a.get("exe") + args.offscreen = bool(a.get("offscreen", True)) + args.port = None + args.log = None + args.wait = 60.0 + args.rest = list(a.get("args", [])) + # cmd_launch prints its own JSON and exits the process on failure, so + # it is the argument shapes that are reused here, not the function. + import io + import contextlib + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + guictl.cmd_launch(args) + return [{"type": "text", "text": buf.getvalue().strip()}], False + + if name == "gui_screenshot": + # Downscaled and JPEG by default: a 1600x950 PNG is 2 MB of base64, + # which costs more context than the picture is worth. + fmt = a.get("format", "jpg") + img = guictl.call("/ui/screenshot", + {"width": int(a.get("width", 1280)), "format": fmt, + "quality": int(a.get("quality", 80))}, raw=True) + content = [{"type": "image", + "data": base64.b64encode(img).decode("ascii"), + "mimeType": "image/png" if fmt == "png" else "image/jpeg"}] + if a.get("path"): + with open(a["path"], "wb") as f: + f.write(img) + content.append({"type": "text", "text": a["path"]}) + return content, False + + routes = { + "gui_state": ("/ui/state", lambda a: {}), + "gui_tree": ("/ui/tree", lambda a: { + k: v for k, v in (("q", a.get("q")), ("window", a.get("window")), + ("named", "0" if a.get("all") else "1")) + if v is not None}), + "gui_click": ("/ui/click", lambda a: dict( + point_args(a), button=a.get("button", 0), + double="1" if a.get("double") else "0")), + "gui_move": ("/ui/move", point_args), + "gui_drag": ("/ui/drag", lambda a: { + "from": a["from"], "to": a["to"], "button": a.get("button", 0), + "steps": a.get("steps", 8)}), + "gui_scroll": ("/ui/scroll", lambda a: dict( + point_args(a), dy=a.get("dy", -1.0))), + "gui_key": ("/ui/key", lambda a: {"keys": a["keys"]}), + "gui_text": ("/ui/text", lambda a: dict( + point_args(a), value=a["value"], + enter="1" if a.get("enter", True) else "0")), + "gui_wait": ("/ui/wait", lambda a: {"frames": a.get("frames", 4)}), + } + if name not in routes: + return [{"type": "text", "text": "unknown tool: " + name}], True + path, build = routes[name] + body = guictl.call(path, build(a)) + return [{"type": "text", + "text": json.dumps(body, ensure_ascii=False, indent=2)}], False + + +def handle(msg): + """The response to one request, or None for a notification.""" + method = msg.get("method") + if method == "initialize": + want = (msg.get("params") or {}).get("protocolVersion") or PROTOCOL + return {"protocolVersion": want, + "capabilities": {"tools": {}}, + "serverInfo": {"name": "spirula-gui", "version": "1"}} + if method == "tools/list": + return {"tools": TOOLS} + if method == "tools/call": + params = msg.get("params") or {} + try: + content, is_error = run_tool(params.get("name"), + params.get("arguments") or {}) + except SystemExit as e: + content, is_error = [{"type": "text", "text": str(e)}], True + except Exception as e: # noqa: BLE001 + content, is_error = [{"type": "text", + "text": "%s: %s" % (type(e).__name__, e)}], True + return {"content": content, "isError": is_error} + if method == "ping": + return {} + return None + + +def main(): + out = sys.stdout + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except ValueError: + continue + if "id" not in msg: + continue # a notification + try: + result = handle(msg) + except Exception as e: # noqa: BLE001 + reply = {"jsonrpc": "2.0", "id": msg["id"], + "error": {"code": -32603, "message": str(e)}} + else: + if result is None: + reply = {"jsonrpc": "2.0", "id": msg["id"], + "error": {"code": -32601, + "message": "method not found: %s" + % msg.get("method")}} + else: + reply = {"jsonrpc": "2.0", "id": msg["id"], "result": result} + out.write(json.dumps(reply) + "\n") + out.flush() + + +if __name__ == "__main__": + main() diff --git a/tools/guictl.py b/tools/guictl.py new file mode 100755 index 00000000..dfea28f8 --- /dev/null +++ b/tools/guictl.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Drive the Spirula Studio GUI from a shell. + +A thin client for the loopback control surface in src/app/gui/Automation.cpp, +which the GUI opens when SS_GUI_AUTOMATION=1. Widgets are addressed by their +i18n message name -- the part after "###" in an ImGui label -- so a script +does not care which language the window is in. + + python3 tools/guictl.py launch --offscreen + python3 tools/guictl.py tree -q open + python3 tools/guictl.py click open_dataset + python3 tools/guictl.py shot /tmp/gui.png + +Design and endpoint reference: docs/notes/gui-automation.md +""" + +import argparse +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +def config_dir(): + base = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config") + current = os.path.join(base, "spirula-studio") + legacy = os.path.join(base, "spirulae-splat") + if not os.path.isdir(current) and os.path.isdir(legacy): + return legacy + return current + + +def endpoint(): + """(base url, token), from the file the GUI writes when it starts.""" + host = os.environ.get("SS_GUI_AUTOMATION_HOST", "127.0.0.1") + port = os.environ.get("SS_GUI_AUTOMATION_PORT") + token = os.environ.get("SS_GUI_AUTOMATION_TOKEN", "") + path = os.path.join(config_dir(), "automation.json") + try: + with open(path) as f: + saved = json.load(f) + port = port or str(saved.get("port", 7777)) + token = token or saved.get("token", "") + except (OSError, ValueError): + port = port or "7777" + return "http://%s:%s" % (host, port), token + + +def call(path, params=None, raw=False, timeout=30.0): + base, token = endpoint() + q = dict(params or {}) + if token: + q["token"] = token + url = "%s%s?%s" % (base, path, urllib.parse.urlencode(q)) + try: + with urllib.request.urlopen(url, timeout=timeout) as r: + body = r.read() + except urllib.error.HTTPError as e: + body = e.read() + sys.stderr.write(body.decode("utf-8", "replace") + "\n") + sys.exit(1) + except urllib.error.URLError as e: + sys.stderr.write("cannot reach %s: %s\n" % (base, e.reason)) + sys.stderr.write("is the GUI running with SS_GUI_AUTOMATION=1? " + "(guictl.py launch)\n") + sys.exit(2) + if raw: + return body + return json.loads(body.decode("utf-8")) + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + +def target(args): + """id= / at= for the commands that act somewhere.""" + if getattr(args, "at", None): + return {"at": args.at} + return {"id": args.id, "index": args.index} + + +def cmd_launch(args): + exe = args.exe or os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "build_cuda", "spirula") + exe = os.path.abspath(exe) + if not os.path.exists(exe): + sys.exit("no such executable: %s" % exe) + env = dict(os.environ) + env["SS_GUI_AUTOMATION"] = "1" + if args.port: + env["SS_GUI_AUTOMATION_PORT"] = str(args.port) + if args.offscreen: + env["SS_GUI_OFFSCREEN"] = "1" + log = open(args.log, "ab") if args.log else subprocess.DEVNULL + rest = args.rest[1:] if args.rest[:1] == ["--"] else args.rest + p = subprocess.Popen([exe] + rest, env=env, stdout=log, stderr=log, + start_new_session=True) + # The token file is written once the server is bound; polling /ui/state is + # what actually says the first frame has been drawn. + deadline = time.time() + args.wait + while time.time() < deadline: + if p.poll() is not None: + sys.exit("the GUI exited with code %s" % p.returncode) + try: + base, token = endpoint() + q = urllib.parse.urlencode({"token": token} if token else {}) + with urllib.request.urlopen("%s/ui/state?%s" % (base, q), timeout=1): + print(json.dumps({"ok": True, "pid": p.pid})) + return + except Exception: + time.sleep(0.25) + sys.exit("the GUI did not answer within %gs" % args.wait) + + +def cmd_state(args): + print(json.dumps(call("/ui/state"), indent=2, ensure_ascii=False)) + + +def cmd_tree(args): + params = {"named": "0" if args.all else "1"} + if args.q: + params["q"] = args.q + if args.window: + params["window"] = args.window + items = call("/ui/tree", params)["items"] + if args.json: + print(json.dumps(items, indent=2, ensure_ascii=False)) + return + for it in items: + x0, y0, x1, y1 = it["rect"] + print("%-38s %-28s %4.0f,%-4.0f %s" + % (it["id"], it["label"][:28], (x0 + x1) / 2, (y0 + y1) / 2, + it["window"])) + print("(%d items)" % len(items), file=sys.stderr) + + +def cmd_click(args): + p = target(args) + p["button"] = args.button + p["double"] = "1" if args.double else "0" + print(json.dumps(call("/ui/click", p))) + + +def cmd_move(args): + print(json.dumps(call("/ui/move", target(args)))) + + +def cmd_drag(args): + print(json.dumps(call("/ui/drag", {"from": args.start, "to": args.end, + "button": args.button, + "steps": args.steps}))) + + +def cmd_scroll(args): + p = target(args) + p["dy"] = args.dy + print(json.dumps(call("/ui/scroll", p))) + + +def cmd_key(args): + print(json.dumps(call("/ui/key", {"keys": args.keys}))) + + +def cmd_text(args): + p = target(args) + p["value"] = args.value + p["enter"] = "0" if args.no_enter else "1" + print(json.dumps(call("/ui/text", p))) + + +def cmd_wait(args): + print(json.dumps(call("/ui/wait", {"frames": args.frames}))) + + +def cmd_shot(args): + params = {"format": args.format, "quality": args.quality} + if args.width: + params["width"] = args.width + img = call("/ui/screenshot", params, raw=True) + with open(args.out, "wb") as f: + f.write(img) + print(json.dumps({"ok": True, "path": args.out, "bytes": len(img)})) + + +def cmd_script(args): + """One command per line: `click open_dataset`, `key Escape`, `wait 4`.""" + src = sys.stdin if args.file == "-" else open(args.file) + for raw in src: + line = raw.split("#", 1)[0].strip() + if not line: + continue + print("+", line, file=sys.stderr) + main(line.split()) + + +# --------------------------------------------------------------------------- + +def add_target(p): + p.add_argument("id", nargs="?", help="widget id (the i18n message name)") + p.add_argument("--at", help="x,y in ImGui display coordinates instead") + p.add_argument("--index", type=int, default=0, + help="which of several items sharing the id") + + +def build_parser(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("launch", help="start the GUI with automation armed") + p.add_argument("--exe", help="path to the spirula binary") + p.add_argument("--offscreen", action="store_true") + p.add_argument("--port", type=int) + p.add_argument("--log", help="append the GUI's stdout/stderr here") + p.add_argument("--wait", type=float, default=30.0) + p.add_argument("rest", nargs=argparse.REMAINDER) + p.set_defaults(func=cmd_launch) + + p = sub.add_parser("state", help="screen, phase, queue depth, sizes") + p.set_defaults(func=cmd_state) + + p = sub.add_parser("tree", help="the widgets on screen") + p.add_argument("-q", help="substring of the id or the visible label") + p.add_argument("--window") + p.add_argument("--all", action="store_true", help="include unnamed items") + p.add_argument("--json", action="store_true") + p.set_defaults(func=cmd_tree) + + p = sub.add_parser("click") + add_target(p) + p.add_argument("--button", type=int, default=0, help="0 left 1 right 2 mid") + p.add_argument("--double", action="store_true") + p.set_defaults(func=cmd_click) + + p = sub.add_parser("move", help="hover, e.g. to raise a tooltip") + add_target(p) + p.set_defaults(func=cmd_move) + + p = sub.add_parser("drag", help="press, move, release -- viewport gestures") + p.add_argument("start", help="x,y") + p.add_argument("end", help="x,y") + p.add_argument("--button", type=int, default=0) + p.add_argument("--steps", type=int, default=8) + p.set_defaults(func=cmd_drag) + + p = sub.add_parser("scroll") + add_target(p) + p.add_argument("--dy", type=float, default=-1.0) + p.set_defaults(func=cmd_scroll) + + p = sub.add_parser("key", help='a chord, e.g. "Ctrl+S" or "Escape"') + p.add_argument("keys") + p.set_defaults(func=cmd_key) + + p = sub.add_parser("text", help="focus an input and replace its contents") + add_target(p) + p.add_argument("value") + p.add_argument("--no-enter", action="store_true") + p.set_defaults(func=cmd_text) + + p = sub.add_parser("wait", help="let N frames pass") + p.add_argument("--frames", type=int, default=4) + p.set_defaults(func=cmd_wait) + + p = sub.add_parser("shot", help="write the framebuffer to an image file") + p.add_argument("out") + p.add_argument("--width", type=int, help="downscale to this width") + p.add_argument("--format", choices=["png", "jpg"], default="png") + p.add_argument("--quality", type=int, default=85, help="jpg only") + p.set_defaults(func=cmd_shot) + + p = sub.add_parser("script", help="run one command per line") + p.add_argument("file", nargs="?", default="-") + p.set_defaults(func=cmd_script) + return ap + + +def main(argv=None): + args = build_parser().parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main()