mirror of
https://github.com/harry7557558/spirula-studio.git
synced 2026-10-02 02:44:54 +08:00
add MCP server + GUI editing plan
This commit is contained in:
Harry Chen
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"spirula-gui": {
|
||||
"command": "python3",
|
||||
"args": ["tools/gui_mcp.py"]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,10 @@ assets/fonts/ the five embedded UI faces + the full-CJK face
|
||||
table (assets/fonts/README.md). The CJK subsets
|
||||
are GENERATED from the catalogs -- see below
|
||||
tools/codegen/ the four codegen tools (see "Codegen" below)
|
||||
tools/guictl.py drive the GUI from a script -- list the widgets on
|
||||
screen, click them, read the framebuffer back.
|
||||
tools/gui_mcp.py is the same surface as an MCP
|
||||
server; docs/notes/gui-automation.md
|
||||
reference/scripts/ dataset preprocessing CLI tools (Python, standalone;
|
||||
mask.py is embedded into the GUI binary)
|
||||
reference/python/ hand-run tools on NO code path: eval_lpips.py,
|
||||
|
||||
@@ -170,6 +170,10 @@ if(SS_BUILD_GUI)
|
||||
)
|
||||
target_compile_options(imgui_glfw PRIVATE
|
||||
$<$<COMPILE_LANGUAGE:CXX>:${SPLAT_CXX_FLAGS}>)
|
||||
# The item hooks src/app/gui/Automation.cpp implements: one never-taken
|
||||
# branch per widget until it arms them. Not an option -- imgui references
|
||||
# the hooks once this is on, so a build without Automation.cpp would fail.
|
||||
target_compile_definitions(imgui_glfw PUBLIC IMGUI_ENABLE_TEST_ENGINE)
|
||||
target_include_directories(imgui_glfw PUBLIC
|
||||
${imgui_SOURCE_DIR}
|
||||
${imgui_SOURCE_DIR}/backends
|
||||
|
||||
@@ -17,6 +17,8 @@ the detail.
|
||||
| [notes/compare-view.md](notes/compare-view.md) | showing several models at once: engine scene slots, the shared navigation frame |
|
||||
| [notes/vram-splat-x-img.md](notes/vram-splat-x-img.md) | what the largest scratch category costs per element, the bitmask compaction, and the measured dead ends |
|
||||
| [notes/color-transfer.md](notes/color-transfer.md) | linear storage vs the output tone curve, the dynamic range a curve buys, and why its clip is straight-through |
|
||||
| [notes/gui-editing-plan.md](notes/gui-editing-plan.md) | editing in the GUI: the selection seam every tool shares, transforms, mask editing, trajectories, and the order to build them in |
|
||||
| [notes/gui-automation.md](notes/gui-automation.md) | driving the GUI from a script: the imgui item hooks, the loopback control surface, `tools/guictl.py` and the MCP server |
|
||||
| [notes/sfm-in-process-plan.md](notes/sfm-in-process-plan.md) | running SfM inside the GUI: the library seam, the input manifest, and what manual ties need |
|
||||
| [notes/](notes/) | design notes for individual subsystems |
|
||||
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
# Driving the GUI from outside it
|
||||
|
||||
`src/app/gui/Automation.{h,cpp}` opens a loopback HTTP surface that lists the
|
||||
widgets currently on screen, injects mouse and keyboard input at them, and
|
||||
hands back the framebuffer as an image. `tools/guictl.py` is the shell client
|
||||
and `tools/gui_mcp.py` the same thing as an MCP server.
|
||||
|
||||
It exists because the editing work
|
||||
([gui-editing-plan.md](gui-editing-plan.md)) is interaction code — a drag has
|
||||
to produce the selection the user meant — and the only way to check that used
|
||||
to be a human at the keyboard. It is also the cheapest regression test this
|
||||
GUI has ever had: a script and a screenshot.
|
||||
|
||||
## Why not the usual approaches
|
||||
|
||||
Screen automation from outside the process (`xdotool` and friends) needs a real
|
||||
display server, knows nothing about widgets, and on Wayland is mostly blocked
|
||||
by design. Dear ImGui's own test engine does all of this properly, but it is a
|
||||
separate dependency under its own licence, and the part of it that matters here
|
||||
is three functions.
|
||||
|
||||
Those three functions are the ones ImGui already calls: build with
|
||||
`IMGUI_ENABLE_TEST_ENGINE` and `ItemAdd()` reports every widget's id and
|
||||
rectangle, the widgets report their label and status flags, and the hooks are
|
||||
ours to implement. That is the whole item registry, from four symbols and no
|
||||
call-site changes.
|
||||
|
||||
It costs one never-taken branch per widget per frame while
|
||||
`ctx->TestEngineHookItems` is false, which is every run that did not ask for
|
||||
this.
|
||||
|
||||
## Addressing a widget
|
||||
|
||||
The GUI hands ImGui `"text###<message name>"` for everything with an id
|
||||
(`src/app/gui/Ui.h`), so that a language switch does not change a widget's
|
||||
identity and collapse every open header. The same property is what makes
|
||||
scripting it pleasant: the part after `###` is the i18n message name, it is the
|
||||
same in all thirteen languages, and it is a name a person can read.
|
||||
|
||||
```
|
||||
$ python3 tools/guictl.py tree -q open
|
||||
home_open_dataset Open a Dataset... 800,264 ##host/##home_...
|
||||
home_open_splat View a Trained Model 800,348 ##host/##home_...
|
||||
$ python3 tools/guictl.py click home_open_splat
|
||||
```
|
||||
|
||||
Widgets ImGui was given a hidden label (`"##fov"`) are addressable under that
|
||||
spelling. Anything with no id at all — plain text, custom drawing, the viewport
|
||||
image — is reached by coordinates instead, which is also how the viewport
|
||||
gestures are driven.
|
||||
|
||||
## How a click happens
|
||||
|
||||
Injecting input is not one event. ImGui trickles its input queue so that a
|
||||
mouse move and the button press after it land on *different* frames, and an
|
||||
item is only hovered on the frame after the pointer reaches it. So a command
|
||||
expands into a short script of steps, and the frame loop applies exactly one
|
||||
step per frame: move, move, press, release, settle.
|
||||
|
||||
`begin_frame()` runs between `ImGui_ImplGlfw_NewFrame()` and
|
||||
`ImGui::NewFrame()` — after the backend's own mouse update, so an injected
|
||||
position is the later event and wins. `end_frame()` runs after the draw data is
|
||||
rendered and before the buffers are swapped, which is where the item table for
|
||||
the finished frame is published and where a screenshot reads the back buffer.
|
||||
|
||||
An HTTP request blocks until its last step has been applied and two more frames
|
||||
have been drawn, so a command returns when its effect is on screen. `nowait=1`
|
||||
opts out.
|
||||
|
||||
Two smaller details, both of which were bugs first: `ConfigDebugIgnoreFocusLoss`
|
||||
is set, because a driven window is rarely the focused one and ImGui otherwise
|
||||
clears the input state it was just handed; and a frame that applied a step
|
||||
counts as busy, so a scripted run paces at the busy frame rate instead of the
|
||||
15 Hz idle one.
|
||||
|
||||
## Running it
|
||||
|
||||
```bash
|
||||
python3 tools/guictl.py launch --offscreen # or: SS_GUI_AUTOMATION=1 ./build_cuda/spirula
|
||||
python3 tools/guictl.py state
|
||||
python3 tools/guictl.py tree -q mesh
|
||||
python3 tools/guictl.py click menu_file
|
||||
python3 tools/guictl.py key Escape
|
||||
python3 tools/guictl.py drag 800,600 1000,550 # orbit the viewport
|
||||
python3 tools/guictl.py text '##fov' 120
|
||||
python3 tools/guictl.py shot out.png --width 1280 --format jpg
|
||||
python3 tools/guictl.py script my-run.txt # one command per line
|
||||
```
|
||||
|
||||
`launch` passes any trailing arguments to the binary, which takes file paths
|
||||
exactly as a drag-and-drop would — the shortest way to get a model open.
|
||||
|
||||
| variable | |
|
||||
|---|---|
|
||||
| `SS_GUI_AUTOMATION=1` | arm it; without this nothing binds and no hook runs |
|
||||
| `SS_GUI_AUTOMATION_PORT` | default 7777 |
|
||||
| `SS_GUI_AUTOMATION_TOKEN` | fixed token instead of a fresh random one |
|
||||
| `SS_GUI_OFFSCREEN=1` | create the window invisible (still a real GL context) |
|
||||
|
||||
The server binds `127.0.0.1` only and requires a token. That is not paranoia
|
||||
about the network: these are GETs with side effects, and any page in any
|
||||
browser can issue one at localhost. The token is written to
|
||||
`<config dir>/automation.json`, which is where both clients read it from.
|
||||
|
||||
Offscreen still needs a display to create a context against. For a machine with
|
||||
none, run it under `Xvfb`; the rest behaves identically.
|
||||
|
||||
## Endpoints
|
||||
|
||||
All GET, all under `/ui/`, all taking `token=`. `id=` names a widget and
|
||||
`at=x,y` a point in ImGui display coordinates; `index=` picks between items
|
||||
sharing an id.
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `/ui/state` | frame counter, item count, queue depth, display and framebuffer size, plus what `GuiApp::state_json()` reports: screen, training phase and step, whether a dialog is open, how many models are loaded |
|
||||
| `/ui/tree` | the widgets of the last finished frame: `q=` substring, `window=`, `named=0` to include unnamed items |
|
||||
| `/ui/click` | `button=` 0/1/2, `double=1`, `settle=` |
|
||||
| `/ui/move` | hover, for tooltips and hover-only state |
|
||||
| `/ui/drag` | `from=`, `to=`, `steps=` — more steps for a path a tool has to follow |
|
||||
| `/ui/scroll` | `dy=` |
|
||||
| `/ui/key` | `keys=Ctrl+Shift+A` |
|
||||
| `/ui/text` | focus, select all, type `value=`, `enter=0` to leave it open |
|
||||
| `/ui/wait` | `frames=` |
|
||||
| `/ui/screenshot` | `width=` to downscale (area average), `format=jpg`, `quality=`, `path=` to write it server-side |
|
||||
|
||||
Display coordinates are not framebuffer pixels on a HiDPI screen; `/ui/state`
|
||||
reports both and the scale between them.
|
||||
|
||||
## As an MCP server
|
||||
|
||||
`tools/gui_mcp.py` wraps the same endpoints as MCP tools over stdio, in the
|
||||
standard library alone — no dependency, and MCP over stdio is newline-delimited
|
||||
JSON-RPC. `.mcp.json` at the repo root registers it for Claude Code; anything
|
||||
else wants `python3 tools/gui_mcp.py` as the command.
|
||||
|
||||
The reason to prefer it over the shell client is `gui_screenshot`, which
|
||||
returns the picture as an image rather than as a path to one. It downscales to
|
||||
1280 and encodes JPEG by default: the full-size PNG is two megabytes of base64,
|
||||
which costs far more than the picture is worth.
|
||||
|
||||
## What it is not
|
||||
|
||||
It drives the *interface*. It does not know what a tool means, and a test that
|
||||
asserts "this drag selected 412 splats" wants the op-script path in
|
||||
[gui-editing-plan.md](gui-editing-plan.md) instead, which needs no window at
|
||||
all. Use this for the part that genuinely needs a window: that the gesture
|
||||
reaches the tool, that the panel is laid out, that the picture is right.
|
||||
@@ -0,0 +1,358 @@
|
||||
# Editing in the GUI
|
||||
|
||||
A plan for the editing features users keep asking for: placing a model,
|
||||
selecting parts of one and doing something to the selection, cleaning up a
|
||||
model by attribute, fixing a mask by hand, laying out a camera move, and
|
||||
splitting a reconstruction that came out as one model but is two.
|
||||
|
||||
Nothing here is a research problem. Every one of these features is a solved
|
||||
interaction somewhere else — Blender, Photoshop, CloudCompare, MeshLab — and
|
||||
the work is deciding what they share so that the eighth one is cheap rather
|
||||
than an eighth of the code again.
|
||||
|
||||
## The mistake to avoid
|
||||
|
||||
The request reads as seven features. Built as seven features it is seven tool
|
||||
modes, seven undo stories, seven sets of keys, seven translations, and seven
|
||||
places where a later feature does not compose with an earlier one — the point
|
||||
where "select the blue splats *inside this box*" turns out to need a rewrite
|
||||
because the box tool and the colour tool each own a private answer.
|
||||
|
||||
Almost all of it is the same two things over four kinds of document:
|
||||
|
||||
**make a set**, and **do something to the set**.
|
||||
|
||||
Today's list is about ten ways to make a set and about ten things to do with
|
||||
one. As features that is a hundred; across a seam it is twenty, and the
|
||||
twenty-first is a day's work. Everything below follows from taking that seam
|
||||
seriously.
|
||||
|
||||
## The four documents
|
||||
|
||||
| document | element | what already exists |
|
||||
|---|---|---|
|
||||
| sparse reconstruction | 3D point, image, submodel | `ParsedDataset`, `src/sfm/` (incl. `map/Merge.h`) |
|
||||
| splat model | one Gaussian | engine scene slot, `checkpoint/SplatPly.h` |
|
||||
| triangle mesh | vertex, face, component | `meshing::MeshData`, `mesh/MeshImport.cpp` |
|
||||
| image mask | pixel, shape | `app::FrameMask`, `SegmentPanel` |
|
||||
|
||||
Each already has a reader, a renderer and a writer. What none of them has is a
|
||||
**mutable document with a history**. That object is the whole of phase 1, and
|
||||
the four kinds differ only in what an element is.
|
||||
|
||||
## The four objects
|
||||
|
||||
### `EditDoc` — what is open, and what has been done to it
|
||||
|
||||
One per pane. It holds the loaded original, an ordered list of operations, the
|
||||
current selection, and a dirty flag. The decision that matters:
|
||||
|
||||
> An edit is an entry in a list, not a mutation of the loaded data.
|
||||
|
||||
Replaying the list from the original is how undo works, how "save the edits,
|
||||
not the result" works, and how the same edits survive the model being retrained
|
||||
underneath them. A 15M-splat model is about a gigabyte of device memory; a
|
||||
snapshot per undo step is not affordable, and a list of ops is a few hundred
|
||||
bytes per step. Where replay is expensive, keep a periodic snapshot in the
|
||||
on-disk cache — the checkpoint machinery already writes and reads splat PLYs —
|
||||
and replay from the nearest one.
|
||||
|
||||
### `Selection` — a set, owned by the document and not by any tool
|
||||
|
||||
A bit per element, device-resident for the splat and mesh documents, with a
|
||||
host mirror pulled only when something needs a count or a histogram. Make it a
|
||||
`uint8` weight rather than a bit from the start: a soft edge costs the same
|
||||
memory as a hard one, and the training-region-of-interest feature below wants a
|
||||
weight anyway.
|
||||
|
||||
Every tool writes into the *same* selection through a combine mode — replace,
|
||||
add, subtract, intersect — bound to the usual modifiers. That single decision
|
||||
is what makes "box, minus a brush stroke, intersected with *blue and low
|
||||
opacity*" work without the box tool and the colour tool knowing about each
|
||||
other, and it is the reason the histogram selector needs no special case for
|
||||
"inside this region".
|
||||
|
||||
### `Tool` — a modal state machine over viewport input
|
||||
|
||||
Exactly one active at a time:
|
||||
|
||||
```c++
|
||||
struct Tool {
|
||||
virtual void on_enter(EditCtx&) {}
|
||||
virtual Consumed on_event(const InputEvent&, EditCtx&) = 0;
|
||||
virtual void draw_overlay(ImDrawList*, const EditCtx&) {}
|
||||
virtual const Msg& status_hint() const = 0; // the strip at the bottom
|
||||
};
|
||||
```
|
||||
|
||||
Camera navigation is itself the default tool. That is what keeps "does this
|
||||
drag orbit or lasso?" from being a question each feature answers for itself.
|
||||
|
||||
### `Op` — the only thing allowed to change a document
|
||||
|
||||
`apply`, `undo`, a name (a `Msg`, because it appears in the undo menu) and a
|
||||
serialized form. Nothing else writes to the document. An op script is then a
|
||||
file, which is what makes the whole thing testable without a window.
|
||||
|
||||
## Making a set
|
||||
|
||||
**A 2D region, extruded.** Box, ellipse, lasso, polygon and brush are one
|
||||
thing: a screen-space stencil. The tool rasterizes its shape into a bitmask on
|
||||
the CPU — that part is cheap and different per shape — and one kernel does the
|
||||
rest, projecting each element with the current view matrix and testing the
|
||||
bitmask. One kernel, every shape, both backends.
|
||||
|
||||
Two modifiers make it usable rather than a demo: *front-most only*, against the
|
||||
depth the render already produced, and a depth range taken from two clicks.
|
||||
Without them, a lasso around a chair also takes the wall behind it, which is
|
||||
the first thing anyone tries.
|
||||
|
||||
A splat is not a point, so the test needs a policy: by centre, or by any part
|
||||
of the projected extent. The projection kernel already computes that extent;
|
||||
offer both and default to the centre.
|
||||
|
||||
**3D primitives.** An oriented box with a gizmo, a sphere, a half-space from a
|
||||
plane. These are what "crop the scene" actually means, and unlike a screen
|
||||
region they survive a camera move, which makes them the right thing to *store*
|
||||
on a document as a reusable clip.
|
||||
|
||||
**Attribute predicates — the histogram selector.** Any per-element scalar
|
||||
becomes a brushable histogram: opacity, largest and smallest scale, the
|
||||
anisotropy the `erank` regularizer already computes from the scales, the DC
|
||||
colour in a chosen space, distance to the nearest training camera, the
|
||||
accumulated gradient densification already tracks, the observation count of a
|
||||
sparse point. Two of them at once is a 2D density plot with a rubber-band box.
|
||||
|
||||
Two kernels serve all of it: reduce an attribute into 256 bins over an optional
|
||||
mask, and threshold it back into the selection. The attributes go in one table
|
||||
with a name, a getter and a suggested scale, so the panel is generated from the
|
||||
table rather than written once per attribute.
|
||||
|
||||
The "select the blue-white sky splats tangled into the tree branches" case is
|
||||
this composed with a region, and it needs nothing new — paint roughly over the
|
||||
tree with the brush, then *intersect* with a colour and opacity box. That
|
||||
composition is the feature; neither half is.
|
||||
|
||||
**Connected components.** A grid-backed union-find over element positions.
|
||||
This is "remove the floaters" and it is also the machinery the sparse-model
|
||||
split below needs.
|
||||
|
||||
**Grow, shrink, smooth.** A k-NN dilation of the selection. Small, and the
|
||||
difference between a brush selection that is usable and one that is not.
|
||||
|
||||
## Doing something with it
|
||||
|
||||
Delete. Isolate (keep only). Transform. Recolour, set opacity, clamp scale.
|
||||
Export the subset. Lock, so densification and later edits leave it alone.
|
||||
|
||||
Two are worth more than the rest:
|
||||
|
||||
**Assign to a group.** A named, saved selection. This is what "segment the
|
||||
model into components" means in practice, it is what makes a selection
|
||||
reusable, exportable and re-editable, and it gives the histogram and the region
|
||||
tools something to write to that outlives the click.
|
||||
|
||||
**Set a training weight.** The region-of-interest ask is not an edit of a
|
||||
model, it is an input to the *next run*, and it should not be stored in a splat
|
||||
file. A group exported as an index list with a weight, named by a training
|
||||
flag, keeps it where it belongs. Note the constraint that comes with it: the
|
||||
indices only mean anything as long as densification has not renumbered
|
||||
everything, so the weight has to be carried as a spatial region or re-derived
|
||||
per run, not as a list of integers that silently rots.
|
||||
|
||||
## Transform, Blender-style
|
||||
|
||||
Two layers, and they are different features:
|
||||
|
||||
- the **gizmo**: axis and plane handles drawn in the viewport;
|
||||
- the **modal operator**: `G` / `R` / `S`, then `X` / `Y` / `Z` (twice for the
|
||||
complementary plane), typed numbers, `Shift` for precision, `Ctrl` for snap,
|
||||
`Enter` or left-click to confirm, `Esc` or right-click to cancel.
|
||||
|
||||
Ship the modal operator first. It is the grammar experienced users actually
|
||||
want, and it is the *easier* of the two: a small state machine over key events
|
||||
with a live preview and no hit-testing at all.
|
||||
|
||||
Say the pivot and the axis frame once, in the transform context — median point,
|
||||
3D cursor, bounding-box centre, individual origins, in global, local or view
|
||||
axes — and every tool inherits them.
|
||||
|
||||
**Where a transform lives.** `ViewportPanel::set_model_transform` already
|
||||
places a whole model by moving the *camera* instead of the geometry
|
||||
(`docs/notes/compare-view.md`), which costs nothing and is the right preview
|
||||
path. It stops working the moment a transform applies to a subset. So: a
|
||||
whole-model placement stays camera-side until it is saved, and a subset
|
||||
transform is an op that rewrites the elements.
|
||||
|
||||
For splats that rewrite is `mean -> sRx + t`, `quat -> Rq`, `log scale -> +log
|
||||
s` — and the spherical harmonics have to be **rotated** with the model, band by
|
||||
band, or the view-dependent colour swims as the object turns. That is the one
|
||||
part of "just rotate it" that is real work rather than three lines, and it is
|
||||
the same rotation a dataset-level pose normalization would want.
|
||||
|
||||
## The 2D half: masks, the pen tool, intelligent scissors
|
||||
|
||||
`app::FrameMask` is further along than it looks. It already holds an *ordered*
|
||||
list of keep/remove ellipses and rectangles normalized to the frame, plus an
|
||||
image stencil intersected with them, and `SegmentPanel` already drags those
|
||||
shapes over a real decoded frame. Four things are missing:
|
||||
|
||||
- **A path shape.** A closed polygon or Bezier with the same keep/remove
|
||||
semantics, in the same ordered list: one more `MaskShape::Kind`, one more
|
||||
case in `parse_mask_shapes`, and a scanline fill in
|
||||
`rasterize_frame_mask`.
|
||||
- **Livewire (intelligent scissors).** Dijkstra over a cost image built from
|
||||
gradient magnitude and direction; the user drops anchors and the path snaps
|
||||
to the edge between them. It is a couple of hundred lines over an image the
|
||||
panel has already decoded, it runs on the CPU at preview resolution, and the
|
||||
cost image is computed once per frame shown.
|
||||
- **A paint layer.** `FrameMask::image` is already a stencil image, so the
|
||||
storage exists; what is new is a canvas to paint on and a convention for
|
||||
where the PNG is written next to the dataset.
|
||||
- **Refining an AI mask.** The composition rule is already "shapes ∩ image".
|
||||
Keep the model's output immutable and store the corrections as a separate
|
||||
layer composed on top; a correction that is destroyed by re-running the model
|
||||
is a correction nobody will make twice.
|
||||
|
||||
One data-model change is unavoidable: today a stencil belongs to a *camera*,
|
||||
and hand corrections belong to a *frame*. Both have to exist, keyed the way
|
||||
`DatasetPrep` already keys cameras.
|
||||
|
||||
## Camera trajectories and video export
|
||||
|
||||
A trajectory is keyframes (pose, field of view, time, and the render options —
|
||||
a fly-through that changes buffer halfway is a legitimate thing to want), an
|
||||
interpolation (Catmull-Rom on position, slerp/squad on rotation, with an
|
||||
optional constant-speed reparameterization so a dense cluster of keyframes does
|
||||
not crawl), and a render job.
|
||||
|
||||
Almost everything is already here: `NavCamera` for the pose, `RenderWorker` for
|
||||
the frames, `app/WriterPool.h` for encoding them off the render thread, and an
|
||||
ffmpeg dependency the app already shells out to. What is missing is UI — a
|
||||
timeline strip, "key the current view", a curve drawn in the viewport, a scrub
|
||||
— plus a JSON file stored next to the model so the same move can be re-rendered
|
||||
after the model is retrained. The turntable and orbit presets are the same
|
||||
object with the keyframes generated.
|
||||
|
||||
## Splitting and merging a sparse reconstruction
|
||||
|
||||
`src/sfm/map/Merge.h` was written for this: `alignReconstructions`, `mergeInto`
|
||||
and `MergeSession` are separated, in its own words, *because a GUI will drive
|
||||
them individually*. So merge is mostly wiring plus a preview of the alignment
|
||||
before it is committed.
|
||||
|
||||
Split is the new half, and it needs a stated rule rather than an implementation:
|
||||
an image belongs to a submodel as much as a point does, and a cut leaves tracks
|
||||
straddling it. Decide once — a track follows the majority of its observations,
|
||||
ties are dropped — and show the surviving track count *before* the cut is
|
||||
applied. A destructive operation whose result is a number nobody can predict
|
||||
needs that number on screen first.
|
||||
|
||||
## Undo, and what it costs
|
||||
|
||||
Start with the cheap trick: **deletes are soft**. A deleted element is marked,
|
||||
not removed, and compaction happens at save. Undo of a delete is then one bit,
|
||||
hide and isolate become the same op with a different flag, and the machinery is
|
||||
the selection bitset that already exists. The cost is that the document carries
|
||||
its garbage until it is written out, which is the right trade.
|
||||
|
||||
For the ops that cannot be soft, carry the inverse where it is cheap (any
|
||||
transform) and a compact delta where it is not. Size the delta honestly before
|
||||
choosing: one splat at SH degree 3 is 59 floats — 236 bytes — so a one-million
|
||||
splat delta is 236 MB, which belongs in a disk-backed spill, not in RAM behind
|
||||
a menu the user does not know is holding it. Cap the history by count *and* by
|
||||
bytes, show what it is holding, and never let it be the reason a session runs
|
||||
out of memory.
|
||||
|
||||
## Where the code goes
|
||||
|
||||
```
|
||||
src/app/gui/edit/
|
||||
EditDoc.{h,cpp} the document, the op list, undo/redo
|
||||
Selection.{h,cpp} the mask, combine modes, named groups
|
||||
Tool.h the tool interface and the active-tool stack
|
||||
tools/ SelectBox, SelectLasso, SelectBrush, Transform, Pen, ...
|
||||
Ops.{h,cpp} delete, transform, recolour, assign to group, ...
|
||||
Gizmo.{h,cpp}
|
||||
Attributes.{h,cpp} the per-element scalar table + the brushable histogram
|
||||
Trajectory.{h,cpp}
|
||||
src/engine/EngineEdit.cpp engine_select_*, engine_apply_transform, ...
|
||||
src/shaders/select.slang the stencil test and the attribute reduction
|
||||
```
|
||||
|
||||
Rules already in force that this work has to obey, listed because each one is
|
||||
cheaper to follow than to retrofit:
|
||||
|
||||
- Selection and transform kernels are **Slang, on both backends**. Nothing here
|
||||
may become CUDA-only (`AGENTS.md`, "the two-backend rule").
|
||||
- Every visible string is a `Msg`: a tool name, a status hint, and an op name
|
||||
as it appears in the undo menu — which means it is a sentence with a `{0}`,
|
||||
never fragments concatenated.
|
||||
- A **keyboard shortcut is not interface copy**. `G` / `R` / `S` stay `G` /
|
||||
`R` / `S` in every language, exactly as `--sh-degree` does.
|
||||
- Any new setting that gets saved needs its row in the preset field table, or
|
||||
it will save, load, and quietly run at its default.
|
||||
- The engine is a process-global singleton and `CompareView` holds one mutex
|
||||
for every pane. An edit runs under that mutex, between renders, like
|
||||
everything else that touches the engine.
|
||||
|
||||
## Order of work
|
||||
|
||||
Each phase is shippable on its own and reuses the previous seam rather than
|
||||
widening it.
|
||||
|
||||
1. **The spine.** `EditDoc`, `Selection`, `Op`, undo, soft delete, one
|
||||
box-select tool, "delete selection", "save as" — on the splat document in
|
||||
the viewer screen only. This is already the most-asked-for cleanup
|
||||
workflow, and everything later is an addition to a working thing.
|
||||
2. **More ways to select.** Lasso, polygon, brush, invert, grow/shrink, the
|
||||
depth modifiers. Nothing else changes.
|
||||
3. **The histogram panel and named groups.**
|
||||
4. **Transform.** The modal operator, then the gizmo, then SH rotation, then
|
||||
baking a placement on save.
|
||||
5. **The mask editor.** Path shape, livewire, paint layer, per-frame
|
||||
corrections.
|
||||
6. **Trajectories and video export.**
|
||||
7. **Sparse split/merge, and region-of-interest weighting for training.**
|
||||
|
||||
## What will bite
|
||||
|
||||
- **The engine singleton.** A scene-slot model is not the training world. Keep
|
||||
an edit to one from reaching the other, or a run inherits it.
|
||||
- **Editing then continuing training is not free.** A scene slot carries no
|
||||
optimizer state, so resuming training on an edited model means resizing the
|
||||
Adam moments and the densification arrays in the same order as the splats.
|
||||
`checkpoint/Adapt.cpp` already does host-side layout adaptation on resume;
|
||||
that is the code to reuse rather than a second one.
|
||||
- **i18n volume.** An editing UI is a few hundred new messages across thirteen
|
||||
languages, and the embedded CJK faces are subset to the characters the
|
||||
catalogs use — a new character with no font regeneration is a hollow box
|
||||
mid-sentence.
|
||||
- **ImGui identity.** The viewport is one item and every tool overlay shares
|
||||
its ID space; push an ID per tool, or two tools' handles collide.
|
||||
- **Picking.** `RenderWorker` already returns the 3D point under a pixel from
|
||||
the ray-depth channel it downloads anyway. That is the 3D cursor and
|
||||
click-to-place; a second picking path would be a second answer to the same
|
||||
question.
|
||||
- **Mouse conventions.** The viewport already orbits on the left button, which
|
||||
collides with Blender's "left confirms, right cancels" in a modal operator.
|
||||
Settle it once: with no tool active navigation keeps its buttons, and an
|
||||
active tool owns both buttons for its whole lifetime.
|
||||
- **Discoverability.** A modal grammar is invisible. A status strip naming the
|
||||
active tool and its two or three keys is not decoration — for this style of
|
||||
UI it is the feature.
|
||||
|
||||
## Testing it
|
||||
|
||||
Two levels, and the cheaper one should carry most of the coverage.
|
||||
|
||||
An op list is serializable, so "apply this op script to this model and compare
|
||||
the result" is a golden-file test that needs no window and no GPU beyond the
|
||||
one the kernels run on. Every selection producer and every operation is
|
||||
testable that way.
|
||||
|
||||
The interaction itself — that a drag in the viewport produces the selection the
|
||||
user meant — needs the real window, and that is what
|
||||
[gui-automation.md](gui-automation.md) is for: the harness drives the tools
|
||||
through the same input path a user does and hands back the framebuffer, so a
|
||||
tool can be exercised and eyeballed without a human at the keyboard.
|
||||
@@ -0,0 +1,804 @@
|
||||
// Automation.cpp -- see Automation.h.
|
||||
|
||||
#include "app/gui/Automation.h"
|
||||
|
||||
#include "app/AppPaths.h"
|
||||
#include "app/webviewer/HttpServer.h"
|
||||
#include "core/Env.h"
|
||||
#include "app/gui/GlLoader.h"
|
||||
#include "external/stb_image_write.h"
|
||||
|
||||
#include "imgui.h"
|
||||
#include "imgui_internal.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
// ===========================================================================
|
||||
// State
|
||||
// ===========================================================================
|
||||
|
||||
// One widget as the last completed frame saw it. `key` is what follows "###"
|
||||
// in the label -- the i18n message name (src/app/gui/Ui.h), which is the same
|
||||
// in every language and is what a script addresses.
|
||||
struct Item {
|
||||
unsigned id = 0;
|
||||
std::string key;
|
||||
std::string label;
|
||||
std::string window;
|
||||
float x0 = 0, y0 = 0, x1 = 0, y1 = 0;
|
||||
int flags = 0;
|
||||
};
|
||||
|
||||
// One frame's worth of injected input; several make a gesture. Exactly one
|
||||
// is applied per frame, which is what keeps a move and the click after it in
|
||||
// separate frames -- the hover test between them is what makes the click land.
|
||||
struct Step {
|
||||
enum class Kind { None, MousePos, MouseButton, Wheel, Key, Text, Focus };
|
||||
Kind kind = Kind::None;
|
||||
float x = 0, y = 0;
|
||||
int button = 0;
|
||||
bool down = false;
|
||||
std::vector<int> keys; // ImGuiKey
|
||||
std::string text;
|
||||
};
|
||||
|
||||
struct State {
|
||||
bool armed = false;
|
||||
HttpServer http;
|
||||
std::string token;
|
||||
std::function<std::string()> state_source;
|
||||
|
||||
std::mutex mu;
|
||||
std::condition_variable cv;
|
||||
|
||||
std::vector<Item> building; // GUI thread only, the frame in flight
|
||||
std::vector<Item> published; // guarded by mu
|
||||
uint64_t frame_no = 0; // guarded by mu
|
||||
float display_w = 0, display_h = 0, fb_scale = 1;
|
||||
int fb_w = 0, fb_h = 0;
|
||||
|
||||
std::deque<Step> queue; // guarded by mu
|
||||
uint64_t queued = 0, applied = 0;
|
||||
|
||||
bool want_shot = false; // guarded by mu
|
||||
bool shot_ready = false;
|
||||
int shot_w = 0; // 0 = the framebuffer's own width
|
||||
bool shot_jpeg = false;
|
||||
int shot_quality = 85;
|
||||
std::vector<uint8_t> shot;
|
||||
};
|
||||
|
||||
State& st() {
|
||||
static State s;
|
||||
return s;
|
||||
}
|
||||
|
||||
constexpr size_t kMaxItems = 8192;
|
||||
|
||||
// ===========================================================================
|
||||
// JSON
|
||||
// ===========================================================================
|
||||
|
||||
void json_str(std::string& out, const std::string& s) {
|
||||
out += '"';
|
||||
for (unsigned char c : s) {
|
||||
switch (c) {
|
||||
case '"': out += "\\\""; break;
|
||||
case '\\': out += "\\\\"; break;
|
||||
case '\n': out += "\\n"; break;
|
||||
case '\r': out += "\\r"; break;
|
||||
case '\t': out += "\\t"; break;
|
||||
default:
|
||||
if (c < 0x20) {
|
||||
char buf[8];
|
||||
std::snprintf(buf, sizeof(buf), "\\u%04x", c);
|
||||
out += buf;
|
||||
} else {
|
||||
out += (char)c;
|
||||
}
|
||||
}
|
||||
}
|
||||
out += '"';
|
||||
}
|
||||
|
||||
std::string json_num(double v) {
|
||||
char buf[32];
|
||||
std::snprintf(buf, sizeof(buf), "%.2f", v);
|
||||
return buf;
|
||||
}
|
||||
|
||||
HttpResponse ok_json(const std::string& body) { return HttpResponse::json(body); }
|
||||
|
||||
HttpResponse err_json(int status, const std::string& what) {
|
||||
std::string body = "{\"ok\":false,\"error\":";
|
||||
json_str(body, what);
|
||||
body += "}";
|
||||
return HttpResponse::text(status, body, "application/json");
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Keys
|
||||
// ===========================================================================
|
||||
|
||||
struct KeyName { const char* name; ImGuiKey key; };
|
||||
|
||||
const KeyName kKeys[] = {
|
||||
{"tab", ImGuiKey_Tab}, {"left", ImGuiKey_LeftArrow},
|
||||
{"right", ImGuiKey_RightArrow}, {"up", ImGuiKey_UpArrow},
|
||||
{"down", ImGuiKey_DownArrow}, {"pageup", ImGuiKey_PageUp},
|
||||
{"pagedown", ImGuiKey_PageDown}, {"home", ImGuiKey_Home},
|
||||
{"end", ImGuiKey_End}, {"insert", ImGuiKey_Insert},
|
||||
{"delete", ImGuiKey_Delete}, {"backspace", ImGuiKey_Backspace},
|
||||
{"space", ImGuiKey_Space}, {"enter", ImGuiKey_Enter},
|
||||
{"escape", ImGuiKey_Escape}, {"esc", ImGuiKey_Escape},
|
||||
{"ctrl", ImGuiKey_LeftCtrl}, {"shift", ImGuiKey_LeftShift},
|
||||
{"alt", ImGuiKey_LeftAlt}, {"super", ImGuiKey_LeftSuper},
|
||||
{"minus", ImGuiKey_Minus}, {"equal", ImGuiKey_Equal},
|
||||
{"comma", ImGuiKey_Comma}, {"period", ImGuiKey_Period},
|
||||
{"slash", ImGuiKey_Slash},
|
||||
};
|
||||
|
||||
std::string lower(std::string s) {
|
||||
for (char& c : s) c = (char)std::tolower((unsigned char)c);
|
||||
return s;
|
||||
}
|
||||
|
||||
// "Ctrl+Shift+A" -> the key events to hold, innermost last. Empty on a name
|
||||
// this table does not know.
|
||||
bool parse_chord(const std::string& spec, std::vector<int>& out) {
|
||||
size_t pos = 0;
|
||||
while (pos <= spec.size()) {
|
||||
size_t plus = spec.find('+', pos);
|
||||
std::string part = lower(spec.substr(pos, plus == std::string::npos
|
||||
? std::string::npos
|
||||
: plus - pos));
|
||||
if (part.empty()) return false;
|
||||
ImGuiKey k = ImGuiKey_None;
|
||||
for (const KeyName& kn : kKeys)
|
||||
if (part == kn.name) { k = kn.key; break; }
|
||||
if (k == ImGuiKey_None && part.size() == 1) {
|
||||
char c = part[0];
|
||||
if (c >= 'a' && c <= 'z') k = (ImGuiKey)(ImGuiKey_A + (c - 'a'));
|
||||
else if (c >= '0' && c <= '9') k = (ImGuiKey)(ImGuiKey_0 + (c - '0'));
|
||||
}
|
||||
if (k == ImGuiKey_None && part.size() >= 2 && part[0] == 'f') {
|
||||
int n = std::atoi(part.c_str() + 1);
|
||||
if (n >= 1 && n <= 12) k = (ImGuiKey)(ImGuiKey_F1 + (n - 1));
|
||||
}
|
||||
if (k == ImGuiKey_None) return false;
|
||||
out.push_back((int)k);
|
||||
if (plus == std::string::npos) break;
|
||||
pos = plus + 1;
|
||||
}
|
||||
return !out.empty();
|
||||
}
|
||||
|
||||
// The modifier ImGui derives from a physical key, so a chord reaches a
|
||||
// shortcut test as well as a key-down test.
|
||||
ImGuiKey mod_of(int key) {
|
||||
switch ((ImGuiKey)key) {
|
||||
case ImGuiKey_LeftCtrl: case ImGuiKey_RightCtrl: return ImGuiMod_Ctrl;
|
||||
case ImGuiKey_LeftShift: case ImGuiKey_RightShift: return ImGuiMod_Shift;
|
||||
case ImGuiKey_LeftAlt: case ImGuiKey_RightAlt: return ImGuiMod_Alt;
|
||||
case ImGuiKey_LeftSuper: case ImGuiKey_RightSuper: return ImGuiMod_Super;
|
||||
default: return ImGuiKey_None;
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Queue
|
||||
// ===========================================================================
|
||||
|
||||
uint64_t enqueue(const std::vector<Step>& steps) {
|
||||
State& s = st();
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
for (const Step& step : steps) s.queue.push_back(step);
|
||||
s.queued += steps.size();
|
||||
return s.queued;
|
||||
}
|
||||
|
||||
// Blocks until every step queued up to `mark` has been applied and one more
|
||||
// frame has been drawn with the result. False on timeout.
|
||||
bool wait_applied(uint64_t mark, double timeout_s) {
|
||||
State& s = st();
|
||||
std::unique_lock<std::mutex> lk(s.mu);
|
||||
uint64_t want_frame = 0;
|
||||
if (!s.cv.wait_for(lk, std::chrono::duration<double>(timeout_s),
|
||||
[&] { return s.applied >= mark; }))
|
||||
return false;
|
||||
want_frame = s.frame_no + 2;
|
||||
return s.cv.wait_for(lk, std::chrono::duration<double>(timeout_s),
|
||||
[&] { return s.frame_no >= want_frame; });
|
||||
}
|
||||
|
||||
// A move, then the press and the release, each in its own frame. `settle`
|
||||
// frames of nothing follow, which is what lets a popup open before the next
|
||||
// command looks for what is in it.
|
||||
void push_click(std::vector<Step>& out, float x, float y, int button,
|
||||
bool dbl, int settle) {
|
||||
Step move;
|
||||
move.kind = Step::Kind::MousePos;
|
||||
move.x = x; move.y = y;
|
||||
out.push_back(move);
|
||||
out.push_back(move);
|
||||
for (int i = 0; i < (dbl ? 2 : 1); i++) {
|
||||
Step b;
|
||||
b.kind = Step::Kind::MouseButton;
|
||||
b.button = button;
|
||||
b.down = true;
|
||||
out.push_back(b);
|
||||
b.down = false;
|
||||
out.push_back(b);
|
||||
}
|
||||
for (int i = 0; i < settle; i++) out.push_back(Step{});
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Item lookup
|
||||
// ===========================================================================
|
||||
|
||||
bool find_item(const std::string& key, int index, Item& out, std::string& err) {
|
||||
State& s = st();
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
int seen = 0;
|
||||
for (const Item& it : s.published) {
|
||||
if (it.key != key) continue;
|
||||
if (seen++ != index) continue;
|
||||
if (it.x1 <= it.x0 || it.y1 <= it.y0) {
|
||||
err = "item has an empty rect: " + key;
|
||||
return false;
|
||||
}
|
||||
out = it;
|
||||
return true;
|
||||
}
|
||||
err = seen == 0 ? "no item on screen with id: " + key
|
||||
: "only " + std::to_string(seen) + " item(s) with id: " + key;
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ===========================================================================
|
||||
// ImGui item hooks
|
||||
//
|
||||
// Declared by imgui_internal.h under IMGUI_ENABLE_TEST_ENGINE. Nothing runs
|
||||
// until ctx->TestEngineHookItems is set, which only arm() does.
|
||||
// ===========================================================================
|
||||
|
||||
void ImGuiTestEngineHook_ItemAdd(ImGuiContext* ctx, ImGuiID id, const ImRect& bb,
|
||||
const ImGuiLastItemData* item_data) {
|
||||
(void)item_data;
|
||||
State& s = st();
|
||||
if (s.building.size() >= kMaxItems) return;
|
||||
Item it;
|
||||
it.id = (unsigned)id;
|
||||
it.x0 = bb.Min.x; it.y0 = bb.Min.y;
|
||||
it.x1 = bb.Max.x; it.y1 = bb.Max.y;
|
||||
if (ctx->CurrentWindow) it.window = ctx->CurrentWindow->Name;
|
||||
s.building.push_back(std::move(it));
|
||||
}
|
||||
|
||||
void ImGuiTestEngineHook_ItemInfo(ImGuiContext* ctx, ImGuiID id, const char* label,
|
||||
ImGuiItemStatusFlags flags) {
|
||||
(void)ctx;
|
||||
State& s = st();
|
||||
for (size_t i = s.building.size(); i-- > 0; ) {
|
||||
if (s.building[i].id != (unsigned)id) continue;
|
||||
Item& it = s.building[i];
|
||||
it.flags = (int)flags;
|
||||
if (!label) return;
|
||||
const char* hash = std::strstr(label, "###");
|
||||
if (hash) {
|
||||
it.label.assign(label, hash - label);
|
||||
it.key = hash + 3;
|
||||
} else {
|
||||
const char* hide = std::strstr(label, "##");
|
||||
it.label.assign(label, hide ? hide - label : std::strlen(label));
|
||||
it.key = label;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void ImGuiTestEngineHook_Log(ImGuiContext* ctx, const char* fmt, ...) {
|
||||
(void)ctx; (void)fmt;
|
||||
}
|
||||
|
||||
const char* ImGuiTestEngine_FindItemDebugLabel(ImGuiContext* ctx, ImGuiID id) {
|
||||
(void)ctx;
|
||||
State& s = st();
|
||||
for (const Item& it : s.building)
|
||||
if (it.id == (unsigned)id && !it.label.empty()) return it.label.c_str();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Endpoints
|
||||
// ===========================================================================
|
||||
|
||||
namespace {
|
||||
|
||||
std::string state_json() {
|
||||
State& s = st();
|
||||
std::string out = "{\"ok\":true";
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
out += ",\"frame\":" + std::to_string(s.frame_no);
|
||||
out += ",\"items\":" + std::to_string(s.published.size());
|
||||
out += ",\"queued\":" + std::to_string(s.queued - s.applied);
|
||||
out += ",\"display\":[" + json_num(s.display_w) + "," +
|
||||
json_num(s.display_h) + "]";
|
||||
out += ",\"framebuffer\":[" + std::to_string(s.fb_w) + "," +
|
||||
std::to_string(s.fb_h) + "]";
|
||||
out += ",\"fb_scale\":" + json_num(s.fb_scale);
|
||||
}
|
||||
if (s.state_source) {
|
||||
std::string extra = s.state_source();
|
||||
if (!extra.empty()) out += "," + extra;
|
||||
}
|
||||
out += "}";
|
||||
return out;
|
||||
}
|
||||
|
||||
HttpResponse handle_tree(const HttpRequest& r) {
|
||||
const std::string q = lower(r.get("q"));
|
||||
const std::string window = r.get("window");
|
||||
const bool named_only = r.get_bool("named", true);
|
||||
State& s = st();
|
||||
std::string out = "{\"ok\":true,\"items\":[";
|
||||
bool first = true;
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
for (const Item& it : s.published) {
|
||||
if (named_only && it.key.empty()) continue;
|
||||
if (!window.empty() && it.window != window) continue;
|
||||
if (!q.empty() && lower(it.key).find(q) == std::string::npos &&
|
||||
lower(it.label).find(q) == std::string::npos)
|
||||
continue;
|
||||
if (!first) out += ",";
|
||||
first = false;
|
||||
out += "{\"id\":";
|
||||
json_str(out, it.key);
|
||||
out += ",\"label\":";
|
||||
json_str(out, it.label);
|
||||
out += ",\"window\":";
|
||||
json_str(out, it.window);
|
||||
out += ",\"rect\":[" + json_num(it.x0) + "," + json_num(it.y0) + "," +
|
||||
json_num(it.x1) + "," + json_num(it.y1) + "]";
|
||||
out += ",\"flags\":" + std::to_string(it.flags) + "}";
|
||||
}
|
||||
out += "]}";
|
||||
return ok_json(out);
|
||||
}
|
||||
|
||||
// Where a command acts: the centre of the item named by `id`, or the `at=x,y`
|
||||
// the caller gave instead.
|
||||
bool resolve_point(const HttpRequest& r, float& x, float& y, std::string& err) {
|
||||
const std::string at = r.get("at");
|
||||
if (!at.empty()) {
|
||||
if (std::sscanf(at.c_str(), "%f,%f", &x, &y) != 2) {
|
||||
err = "at= wants x,y";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
const std::string id = r.get("id");
|
||||
if (id.empty()) {
|
||||
err = "give id= or at=";
|
||||
return false;
|
||||
}
|
||||
Item it;
|
||||
if (!find_item(id, r.get_int("index", 0), it, err)) return false;
|
||||
x = 0.5f * (it.x0 + it.x1);
|
||||
y = 0.5f * (it.y0 + it.y1);
|
||||
return true;
|
||||
}
|
||||
|
||||
HttpResponse finish(const HttpRequest& r, uint64_t mark) {
|
||||
if (r.get_bool("nowait", false)) return ok_json("{\"ok\":true}");
|
||||
if (!wait_applied(mark, r.get_double("timeout", 15.0)))
|
||||
return err_json(504, "timed out waiting for the frame loop");
|
||||
return ok_json(state_json());
|
||||
}
|
||||
|
||||
HttpResponse handle_click(const HttpRequest& r) {
|
||||
float x = 0, y = 0;
|
||||
std::string err;
|
||||
if (!resolve_point(r, x, y, err)) return err_json(404, err);
|
||||
std::vector<Step> steps;
|
||||
push_click(steps, x, y, r.get_int("button", 0), r.get_bool("double", false),
|
||||
r.get_int("settle", 2));
|
||||
return finish(r, enqueue(steps));
|
||||
}
|
||||
|
||||
HttpResponse handle_move(const HttpRequest& r) {
|
||||
float x = 0, y = 0;
|
||||
std::string err;
|
||||
if (!resolve_point(r, x, y, err)) return err_json(404, err);
|
||||
Step m;
|
||||
m.kind = Step::Kind::MousePos;
|
||||
m.x = x; m.y = y;
|
||||
return finish(r, enqueue({m, Step{}}));
|
||||
}
|
||||
|
||||
HttpResponse handle_drag(const HttpRequest& r) {
|
||||
float x0 = 0, y0 = 0, x1 = 0, y1 = 0;
|
||||
if (std::sscanf(r.get("from").c_str(), "%f,%f", &x0, &y0) != 2 ||
|
||||
std::sscanf(r.get("to").c_str(), "%f,%f", &x1, &y1) != 2)
|
||||
return err_json(400, "drag wants from=x,y and to=x,y");
|
||||
const int n = std::max(1, std::min(256, r.get_int("steps", 8)));
|
||||
const int button = r.get_int("button", 0);
|
||||
std::vector<Step> steps;
|
||||
Step m;
|
||||
m.kind = Step::Kind::MousePos;
|
||||
m.x = x0; m.y = y0;
|
||||
steps.push_back(m);
|
||||
steps.push_back(m);
|
||||
Step b;
|
||||
b.kind = Step::Kind::MouseButton;
|
||||
b.button = button;
|
||||
b.down = true;
|
||||
steps.push_back(b);
|
||||
for (int i = 1; i <= n; i++) {
|
||||
Step p;
|
||||
p.kind = Step::Kind::MousePos;
|
||||
p.x = x0 + (x1 - x0) * (float)i / (float)n;
|
||||
p.y = y0 + (y1 - y0) * (float)i / (float)n;
|
||||
steps.push_back(p);
|
||||
}
|
||||
b.down = false;
|
||||
steps.push_back(b);
|
||||
steps.push_back(Step{});
|
||||
return finish(r, enqueue(steps));
|
||||
}
|
||||
|
||||
HttpResponse handle_scroll(const HttpRequest& r) {
|
||||
float x = 0, y = 0;
|
||||
std::string err;
|
||||
if (!resolve_point(r, x, y, err)) return err_json(404, err);
|
||||
Step m;
|
||||
m.kind = Step::Kind::MousePos;
|
||||
m.x = x; m.y = y;
|
||||
Step w;
|
||||
w.kind = Step::Kind::Wheel;
|
||||
w.x = (float)r.get_double("dx", 0.0);
|
||||
w.y = (float)r.get_double("dy", -1.0);
|
||||
return finish(r, enqueue({m, m, w, Step{}}));
|
||||
}
|
||||
|
||||
HttpResponse handle_key(const HttpRequest& r) {
|
||||
const std::string spec = r.get("keys");
|
||||
if (spec.empty()) return err_json(400, "key wants keys=Ctrl+S");
|
||||
std::vector<int> chord;
|
||||
if (!parse_chord(spec, chord)) return err_json(400, "unknown key: " + spec);
|
||||
Step down, up;
|
||||
down.kind = up.kind = Step::Kind::Key;
|
||||
down.keys = chord;
|
||||
down.down = true;
|
||||
up.keys = chord;
|
||||
up.down = false;
|
||||
return finish(r, enqueue({down, up, Step{}, Step{}}));
|
||||
}
|
||||
|
||||
HttpResponse handle_text(const HttpRequest& r) {
|
||||
float x = 0, y = 0;
|
||||
std::string err;
|
||||
if (!resolve_point(r, x, y, err)) return err_json(404, err);
|
||||
std::vector<Step> steps;
|
||||
push_click(steps, x, y, 0, false, 1);
|
||||
std::vector<int> select_all{(int)ImGuiKey_LeftCtrl, (int)ImGuiKey_A};
|
||||
Step down, up;
|
||||
down.kind = up.kind = Step::Kind::Key;
|
||||
down.keys = select_all;
|
||||
down.down = true;
|
||||
up.keys = select_all;
|
||||
up.down = false;
|
||||
steps.push_back(down);
|
||||
steps.push_back(up);
|
||||
Step t;
|
||||
t.kind = Step::Kind::Text;
|
||||
t.text = r.get("value");
|
||||
steps.push_back(t);
|
||||
steps.push_back(Step{});
|
||||
if (r.get_bool("enter", true)) {
|
||||
std::vector<int> ret{(int)ImGuiKey_Enter};
|
||||
Step ed, eu;
|
||||
ed.kind = eu.kind = Step::Kind::Key;
|
||||
ed.keys = ret;
|
||||
ed.down = true;
|
||||
eu.keys = ret;
|
||||
eu.down = false;
|
||||
steps.push_back(ed);
|
||||
steps.push_back(eu);
|
||||
}
|
||||
steps.push_back(Step{});
|
||||
return finish(r, enqueue(steps));
|
||||
}
|
||||
|
||||
HttpResponse handle_wait(const HttpRequest& r) {
|
||||
const int frames = std::max(0, std::min(600, r.get_int("frames", 2)));
|
||||
std::vector<Step> steps((size_t)frames + 1, Step{});
|
||||
return finish(r, enqueue(steps));
|
||||
}
|
||||
|
||||
// Area average, which is what a screenshot wants: a downscale that samples
|
||||
// would drop every one-pixel line the interface is drawn with.
|
||||
std::vector<uint8_t> box_resize(const std::vector<uint8_t>& src, int sw, int sh,
|
||||
int dw, int dh) {
|
||||
std::vector<uint8_t> dst((size_t)dw * dh * 4);
|
||||
for (int y = 0; y < dh; y++) {
|
||||
const int y0 = (int)((int64_t)y * sh / dh);
|
||||
const int y1 = std::max(y0 + 1, (int)((int64_t)(y + 1) * sh / dh));
|
||||
for (int x = 0; x < dw; x++) {
|
||||
const int x0 = (int)((int64_t)x * sw / dw);
|
||||
const int x1 = std::max(x0 + 1, (int)((int64_t)(x + 1) * sw / dw));
|
||||
int acc[4] = {0, 0, 0, 0};
|
||||
for (int sy = y0; sy < y1; sy++)
|
||||
for (int sx = x0; sx < x1; sx++)
|
||||
for (int c = 0; c < 4; c++)
|
||||
acc[c] += src[((size_t)sy * sw + sx) * 4 + c];
|
||||
const int n = (y1 - y0) * (x1 - x0);
|
||||
for (int c = 0; c < 4; c++)
|
||||
dst[((size_t)y * dw + x) * 4 + c] = (uint8_t)(acc[c] / n);
|
||||
}
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
void png_write_cb(void* ctx, void* data, int size) {
|
||||
auto* out = (std::vector<uint8_t>*)ctx;
|
||||
out->insert(out->end(), (uint8_t*)data, (uint8_t*)data + size);
|
||||
}
|
||||
|
||||
HttpResponse handle_screenshot(const HttpRequest& r) {
|
||||
State& s = st();
|
||||
const bool jpeg = r.get("format", "png") == "jpg" ||
|
||||
r.get("format", "png") == "jpeg";
|
||||
std::vector<uint8_t> png;
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(s.mu);
|
||||
s.shot.clear();
|
||||
s.shot_ready = false;
|
||||
s.shot_w = std::max(0, r.get_int("width", 0));
|
||||
s.shot_jpeg = jpeg;
|
||||
s.shot_quality = std::max(1, std::min(100, r.get_int("quality", 85)));
|
||||
s.want_shot = true;
|
||||
if (!s.cv.wait_for(lk, std::chrono::duration<double>(
|
||||
r.get_double("timeout", 10.0)),
|
||||
[&] { return s.shot_ready; })) {
|
||||
s.want_shot = false;
|
||||
return err_json(504, "the frame loop did not deliver a frame");
|
||||
}
|
||||
png.swap(s.shot);
|
||||
}
|
||||
const std::string path = r.get("path");
|
||||
if (!path.empty()) {
|
||||
std::ofstream f(path, std::ios::binary);
|
||||
if (!f) return err_json(500, "cannot write " + path);
|
||||
f.write((const char*)png.data(), (std::streamsize)png.size());
|
||||
std::string body = "{\"ok\":true,\"bytes\":" +
|
||||
std::to_string(png.size()) + ",\"path\":";
|
||||
json_str(body, path);
|
||||
body += "}";
|
||||
return ok_json(body);
|
||||
}
|
||||
HttpResponse out;
|
||||
out.content_type = jpeg ? "image/jpeg" : "image/png";
|
||||
out.body = std::move(png);
|
||||
return out;
|
||||
}
|
||||
|
||||
// Every route goes through this: the server is on the loopback interface, but
|
||||
// a page in a browser can still reach it, and these are GETs with effects.
|
||||
HttpServer::Handler guard(HttpServer::Handler h) {
|
||||
return [h](const HttpRequest& r) -> HttpResponse {
|
||||
if (!st().token.empty() && r.get("token") != st().token)
|
||||
return err_json(403, "bad or missing token=");
|
||||
return h(r);
|
||||
};
|
||||
}
|
||||
|
||||
std::string make_token() {
|
||||
std::random_device rd;
|
||||
std::uniform_int_distribution<int> d(0, 15);
|
||||
std::string t;
|
||||
for (int i = 0; i < 32; i++) t += "0123456789abcdef"[d(rd)];
|
||||
return t;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ===========================================================================
|
||||
// Frame loop
|
||||
// ===========================================================================
|
||||
|
||||
namespace gui {
|
||||
namespace automation {
|
||||
|
||||
bool armed() { return st().armed; }
|
||||
|
||||
void set_state_source(std::function<std::string()> f) {
|
||||
st().state_source = std::move(f);
|
||||
}
|
||||
|
||||
void arm() {
|
||||
if (!spirula::env_on("GUI_AUTOMATION")) return;
|
||||
State& s = st();
|
||||
|
||||
ImGuiContext* ctx = ImGui::GetCurrentContext();
|
||||
if (!ctx) return;
|
||||
ctx->TestEngineHookItems = true;
|
||||
// A driven window is rarely the focused one, and ImGui clears the input
|
||||
// state it was handed when it believes focus was lost.
|
||||
ImGui::GetIO().ConfigDebugIgnoreFocusLoss = true;
|
||||
|
||||
if (const char* t = spirula::env("GUI_AUTOMATION_TOKEN")) s.token = t;
|
||||
else s.token = make_token();
|
||||
|
||||
const char* port_env = spirula::env("GUI_AUTOMATION_PORT");
|
||||
const int port = port_env ? std::atoi(port_env) : 7777;
|
||||
|
||||
s.http.route("/ui/state", guard([](const HttpRequest&) {
|
||||
return ok_json(state_json());
|
||||
}));
|
||||
s.http.route("/ui/tree", guard(handle_tree));
|
||||
s.http.route("/ui/click", guard(handle_click));
|
||||
s.http.route("/ui/move", guard(handle_move));
|
||||
s.http.route("/ui/drag", guard(handle_drag));
|
||||
s.http.route("/ui/scroll", guard(handle_scroll));
|
||||
s.http.route("/ui/key", guard(handle_key));
|
||||
s.http.route("/ui/text", guard(handle_text));
|
||||
s.http.route("/ui/wait", guard(handle_wait));
|
||||
s.http.route("/ui/screenshot", guard(handle_screenshot));
|
||||
|
||||
try {
|
||||
s.http.start("127.0.0.1", port);
|
||||
} catch (const std::exception& e) {
|
||||
ctx->TestEngineHookItems = false;
|
||||
std::fprintf(stderr, "[automation] not listening: %s\n", e.what());
|
||||
return;
|
||||
}
|
||||
s.armed = true;
|
||||
|
||||
const std::string file = app::config_dir() + "/automation.json";
|
||||
std::ofstream f(file);
|
||||
f << "{\"host\":\"127.0.0.1\",\"port\":" << port << ",\"token\":\""
|
||||
<< s.token << "\"}\n";
|
||||
std::fprintf(stderr, "[automation] http://127.0.0.1:%d token in %s\n",
|
||||
port, file.c_str());
|
||||
}
|
||||
|
||||
bool begin_frame() {
|
||||
State& s = st();
|
||||
if (!s.armed) return false;
|
||||
s.building.clear();
|
||||
|
||||
Step step;
|
||||
bool popped = false;
|
||||
size_t left = 0;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
if (!s.queue.empty()) {
|
||||
step = s.queue.front();
|
||||
s.queue.pop_front();
|
||||
popped = true;
|
||||
}
|
||||
left = s.queue.size();
|
||||
}
|
||||
|
||||
ImGuiIO& io = ImGui::GetIO();
|
||||
switch (step.kind) {
|
||||
case Step::Kind::None: break;
|
||||
case Step::Kind::MousePos: io.AddMousePosEvent(step.x, step.y); break;
|
||||
case Step::Kind::MouseButton:
|
||||
io.AddMouseButtonEvent(step.button, step.down);
|
||||
break;
|
||||
case Step::Kind::Wheel: io.AddMouseWheelEvent(step.x, step.y); break;
|
||||
case Step::Kind::Key: {
|
||||
// Held keys go down outermost-first and come up in the same order,
|
||||
// so a modifier is already down when the key it modifies arrives.
|
||||
for (int k : step.keys) {
|
||||
if (ImGuiKey m = mod_of(k); m != ImGuiKey_None)
|
||||
io.AddKeyEvent(m, step.down);
|
||||
io.AddKeyEvent((ImGuiKey)k, step.down);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Step::Kind::Text:
|
||||
io.AddInputCharactersUTF8(step.text.c_str());
|
||||
break;
|
||||
case Step::Kind::Focus: io.AddFocusEvent(true); break;
|
||||
}
|
||||
|
||||
if (popped) {
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
s.applied++;
|
||||
}
|
||||
s.cv.notify_all();
|
||||
return popped || left > 0;
|
||||
}
|
||||
|
||||
void end_frame(int fb_w, int fb_h) {
|
||||
State& s = st();
|
||||
if (!s.armed) return;
|
||||
|
||||
bool shoot = false;
|
||||
int want_w = 0, quality = 85;
|
||||
bool jpeg = false;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
s.published.swap(s.building);
|
||||
s.frame_no++;
|
||||
const ImGuiIO& io = ImGui::GetIO();
|
||||
s.display_w = io.DisplaySize.x;
|
||||
s.display_h = io.DisplaySize.y;
|
||||
s.fb_w = fb_w;
|
||||
s.fb_h = fb_h;
|
||||
s.fb_scale = io.DisplaySize.x > 0 ? (float)fb_w / io.DisplaySize.x : 1.0f;
|
||||
shoot = s.want_shot;
|
||||
want_w = s.shot_w;
|
||||
jpeg = s.shot_jpeg;
|
||||
quality = s.shot_quality;
|
||||
}
|
||||
s.building.clear();
|
||||
if (!shoot || fb_w <= 0 || fb_h <= 0) {
|
||||
s.cv.notify_all();
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> rgba((size_t)fb_w * fb_h * 4);
|
||||
glPixelStorei(GL_PACK_ALIGNMENT, 1);
|
||||
glReadPixels(0, 0, fb_w, fb_h, GL_RGBA, GL_UNSIGNED_BYTE, rgba.data());
|
||||
// GL reads bottom-up; every PNG reader expects the other one.
|
||||
const size_t row = (size_t)fb_w * 4;
|
||||
std::vector<uint8_t> flip((size_t)fb_w * fb_h * 4);
|
||||
for (int y = 0; y < fb_h; y++)
|
||||
std::memcpy(flip.data() + (size_t)y * row,
|
||||
rgba.data() + (size_t)(fb_h - 1 - y) * row, row);
|
||||
|
||||
int out_w = fb_w, out_h = fb_h;
|
||||
if (want_w > 0 && want_w < fb_w) {
|
||||
out_w = want_w;
|
||||
out_h = std::max(1, (int)((int64_t)fb_h * want_w / fb_w));
|
||||
flip = box_resize(flip, fb_w, fb_h, out_w, out_h);
|
||||
}
|
||||
std::vector<uint8_t> png;
|
||||
if (jpeg)
|
||||
stbi_write_jpg_to_func(png_write_cb, &png, out_w, out_h, 4,
|
||||
flip.data(), quality);
|
||||
else
|
||||
stbi_write_png_to_func(png_write_cb, &png, out_w, out_h, 4,
|
||||
flip.data(), out_w * 4);
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(s.mu);
|
||||
s.shot = std::move(png);
|
||||
s.shot_ready = true;
|
||||
s.want_shot = false;
|
||||
}
|
||||
s.cv.notify_all();
|
||||
}
|
||||
|
||||
void shutdown() {
|
||||
State& s = st();
|
||||
if (!s.armed) return;
|
||||
s.http.stop();
|
||||
s.armed = false;
|
||||
s.cv.notify_all();
|
||||
}
|
||||
|
||||
} // namespace automation
|
||||
} // namespace gui
|
||||
@@ -0,0 +1,40 @@
|
||||
#pragma once
|
||||
|
||||
// Driving this GUI from outside it.
|
||||
//
|
||||
// A loopback HTTP surface that lists the widgets currently on screen, injects
|
||||
// mouse/keyboard input at them, and hands back the framebuffer as a PNG. Off
|
||||
// unless SS_GUI_AUTOMATION is set, and the imgui item hooks it needs stay
|
||||
// switched off with it, so a normal run pays one never-taken branch per item.
|
||||
//
|
||||
// Design, endpoints and the client: docs/notes/gui-automation.md.
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
namespace gui {
|
||||
namespace automation {
|
||||
|
||||
// Reads SS_GUI_AUTOMATION and binds the server. Call after
|
||||
// ImGui::CreateContext() and before the first frame; a bind failure is
|
||||
// reported on stderr and leaves automation off.
|
||||
void arm();
|
||||
bool armed();
|
||||
|
||||
// What /ui/state reports about the application on top of what ImGui knows:
|
||||
// a JSON object body without the braces, or "" for nothing.
|
||||
void set_state_source(std::function<std::string()> f);
|
||||
|
||||
// Between glfwPollEvents() and ImGui::NewFrame(): applies one step of the
|
||||
// queued input. True while anything is still queued, which is what keeps the
|
||||
// frame loop at its busy rate for the length of a script.
|
||||
bool begin_frame();
|
||||
|
||||
// After the draw data has been rendered, with the GL context current:
|
||||
// publishes the frame's item table and fills a pending screenshot request.
|
||||
void end_frame(int fb_w, int fb_h);
|
||||
|
||||
void shutdown();
|
||||
|
||||
} // namespace automation
|
||||
} // namespace gui
|
||||
@@ -2354,6 +2354,36 @@ void GuiApp::handle_dialog_result(const std::vector<std::string>& paths) {
|
||||
// Frame
|
||||
// ===========================================================================
|
||||
|
||||
std::string GuiApp::state_json() {
|
||||
auto quoted = [](const std::string& s) {
|
||||
std::string q = "\"";
|
||||
for (char c : s) {
|
||||
if ((unsigned char)c < 0x20) continue;
|
||||
if (c == '"' || c == '\\') q += '\\';
|
||||
q += c;
|
||||
}
|
||||
return q + "\"";
|
||||
};
|
||||
// Index order is the declaration order of Screen and TrainRunner::Phase.
|
||||
static const char* kScreens[] = {"home", "new_dataset", "train", "viewer",
|
||||
"batch", "mesh"};
|
||||
static const char* kPhases[] = {"idle", "loading", "ready", "load_error",
|
||||
"preparing", "training", "done",
|
||||
"train_error"};
|
||||
std::string out = "\"screen\":\"";
|
||||
out += kScreens[(int)_screen];
|
||||
out += "\",\"train_phase\":\"";
|
||||
out += kPhases[(int)_runner.phase()];
|
||||
out += "\",\"step\":" + std::to_string(_runner.latest_progress().step);
|
||||
out += ",\"busy\":";
|
||||
out += native_work_busy() ? "true" : "false";
|
||||
out += ",\"dialog_open\":";
|
||||
out += _dialog.is_open() ? "true" : "false";
|
||||
out += ",\"models_open\":" + std::to_string(_compare.count());
|
||||
out += ",\"dataset\":" + quoted(_cfg.data);
|
||||
return out;
|
||||
}
|
||||
|
||||
void GuiApp::frame() {
|
||||
// Before the first widget: a style swap halfway through a frame would
|
||||
// measure half the window against one scale and half against the other.
|
||||
|
||||
@@ -72,6 +72,11 @@ public:
|
||||
// Draw one frame (between ImGui::NewFrame and ImGui::Render).
|
||||
void frame();
|
||||
|
||||
// What a script needs to know that is not on screen as a widget: the
|
||||
// screen, what is running, what is open. A JSON object body without the
|
||||
// braces, for gui::automation::set_state_source.
|
||||
std::string state_json();
|
||||
|
||||
// Window close button pressed; may open a confirmation dialog instead
|
||||
// of quitting when training is in flight.
|
||||
void request_close();
|
||||
|
||||
+15
-1
@@ -5,7 +5,9 @@
|
||||
#include "app/Tools.h"
|
||||
#include "i18n/catalog/Log.h"
|
||||
#include "app/AppPaths.h"
|
||||
#include "core/Env.h"
|
||||
#include "app/CrashLog.h"
|
||||
#include "app/gui/Automation.h"
|
||||
#include "app/gui/Fonts.h"
|
||||
#include "app/gui/GuiApp.h"
|
||||
#include "app/gui/Layout.h"
|
||||
@@ -179,6 +181,11 @@ int spirula_gui_main(int argc, char** argv) {
|
||||
if (mw > 0) win_w = std::min(win_w, mw);
|
||||
if (mh > 0) win_h = std::min(win_h, mh);
|
||||
}
|
||||
// A driven window still needs a real GL context, but nothing needs it on
|
||||
// screen -- and one that is not on screen cannot be clicked on by accident.
|
||||
if (spirula::env_on("GUI_OFFSCREEN"))
|
||||
glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE);
|
||||
|
||||
// The title is set from the catalog below, once GuiApp has settled the
|
||||
// language; this is only what the window is born with.
|
||||
GLFWwindow* window = glfwCreateWindow(win_w, win_h, "Spirula Studio",
|
||||
@@ -209,9 +216,11 @@ int spirula_gui_main(int argc, char** argv) {
|
||||
|
||||
ImGui_ImplGlfw_InitForOpenGL(window, true);
|
||||
ImGui_ImplOpenGL3_Init(glsl_version);
|
||||
gui::automation::arm();
|
||||
|
||||
{
|
||||
gui::GuiApp app;
|
||||
gui::automation::set_state_source([&app] { return app.state_json(); });
|
||||
app.set_dpi_scale(layout_dpi(window));
|
||||
// The atlas has to exist before the first frame, and the language it
|
||||
// depends on is only settled once GuiApp has read its settings file.
|
||||
@@ -274,9 +283,12 @@ int spirula_gui_main(int argc, char** argv) {
|
||||
|
||||
ImGui_ImplOpenGL3_NewFrame();
|
||||
ImGui_ImplGlfw_NewFrame();
|
||||
// After the backend's own mouse update, so an injected position
|
||||
// is the later event and wins for this frame.
|
||||
const bool scripted = gui::automation::begin_frame();
|
||||
ImGui::NewFrame();
|
||||
app.frame();
|
||||
const bool busy = ui_busy();
|
||||
const bool busy = ui_busy() || scripted;
|
||||
ImGui::Render();
|
||||
|
||||
const double now = glfwGetTime();
|
||||
@@ -293,6 +305,7 @@ int spirula_gui_main(int argc, char** argv) {
|
||||
glClearColor(0.07f, 0.07f, 0.08f, 1.0f);
|
||||
glClear(GL_COLOR_BUFFER_BIT);
|
||||
ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
|
||||
gui::automation::end_frame(w, h); // reads the back buffer
|
||||
glfwSwapBuffers(window);
|
||||
}
|
||||
// Joins worker threads (finishing a final checkpoint save if a stop
|
||||
@@ -301,6 +314,7 @@ int spirula_gui_main(int argc, char** argv) {
|
||||
app.shutdown();
|
||||
}
|
||||
|
||||
gui::automation::shutdown();
|
||||
ImGui_ImplOpenGL3_Shutdown();
|
||||
ImGui_ImplGlfw_Shutdown();
|
||||
ImGui::DestroyContext();
|
||||
|
||||
Executable
+302
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env python3
|
||||
"""MCP stdio server over the GUI control surface.
|
||||
|
||||
The same endpoints tools/guictl.py calls, exposed as MCP tools so a coding
|
||||
agent can list the widgets on screen, click them, and get the framebuffer back
|
||||
as an image without shelling out and reading files.
|
||||
|
||||
Standard library only, on purpose: this repo takes no Python dependencies, and
|
||||
MCP over stdio is newline-delimited JSON-RPC 2.0.
|
||||
|
||||
Register it for Claude Code with .mcp.json at the repo root; anything else
|
||||
that speaks MCP wants `python3 tools/gui_mcp.py` as the stdio command.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import guictl # noqa: E402
|
||||
|
||||
PROTOCOL = "2025-06-18"
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"name": "gui_launch",
|
||||
"description": "Start the Spirula Studio GUI with the automation "
|
||||
"surface armed. Returns once it answers.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"offscreen": {"type": "boolean",
|
||||
"description": "Create the window invisible."},
|
||||
"exe": {"type": "string",
|
||||
"description": "Path to the spirula binary; defaults "
|
||||
"to build_cuda/spirula."},
|
||||
"args": {"type": "array", "items": {"type": "string"},
|
||||
"description": "Files to open, as a drop would."},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_state",
|
||||
"description": "Screen, training phase, dialog and queue state, "
|
||||
"display and framebuffer sizes.",
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
},
|
||||
{
|
||||
"name": "gui_tree",
|
||||
"description": "The widgets on screen: id (the i18n message name, "
|
||||
"stable across languages), visible label, rect, window.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"q": {"type": "string",
|
||||
"description": "Substring of the id or the label."},
|
||||
"window": {"type": "string"},
|
||||
"all": {"type": "boolean",
|
||||
"description": "Include items with no id of their own."},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_click",
|
||||
"description": "Click a widget by id, or a point with at=\"x,y\".",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"at": {"type": "string", "description": "x,y"},
|
||||
"button": {"type": "integer",
|
||||
"description": "0 left, 1 right, 2 middle."},
|
||||
"double": {"type": "boolean"},
|
||||
"index": {"type": "integer",
|
||||
"description": "Which of several items sharing an id."},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_move",
|
||||
"description": "Move the pointer onto a widget or point (hover, "
|
||||
"tooltips).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"id": {"type": "string"}, "at": {"type": "string"},
|
||||
"index": {"type": "integer"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_drag",
|
||||
"description": "Press, move and release: viewport orbit/pan, sliders, "
|
||||
"and the selection gestures.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from": {"type": "string", "description": "x,y"},
|
||||
"to": {"type": "string", "description": "x,y"},
|
||||
"button": {"type": "integer"},
|
||||
"steps": {"type": "integer",
|
||||
"description": "Intermediate positions; more for a "
|
||||
"path a tool has to follow."},
|
||||
},
|
||||
"required": ["from", "to"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_scroll",
|
||||
"description": "Wheel over a widget or point (zoom, scrolling panels).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"id": {"type": "string"}, "at": {"type": "string"},
|
||||
"dy": {"type": "number"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_key",
|
||||
"description": "A key chord, e.g. \"Escape\", \"Ctrl+S\", "
|
||||
"\"Ctrl+Shift+A\".",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"keys": {"type": "string"}},
|
||||
"required": ["keys"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_text",
|
||||
"description": "Focus a text field, select everything in it, and type "
|
||||
"a replacement.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"at": {"type": "string"},
|
||||
"value": {"type": "string"},
|
||||
"enter": {"type": "boolean",
|
||||
"description": "Press Enter afterwards (default true)."},
|
||||
},
|
||||
"required": ["value"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_wait",
|
||||
"description": "Let N frames pass, for work that finishes on the GUI "
|
||||
"thread.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"frames": {"type": "integer"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "gui_screenshot",
|
||||
"description": "The current framebuffer, as an image.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"width": {"type": "integer",
|
||||
"description": "Downscale to this width (default 1280)."},
|
||||
"format": {"type": "string", "enum": ["jpg", "png"],
|
||||
"description": "Default jpg; png for exact pixels."},
|
||||
"quality": {"type": "integer", "description": "jpg, 1-100."},
|
||||
"path": {"type": "string",
|
||||
"description": "Also write the image here."},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def point_args(a):
|
||||
out = {}
|
||||
if a.get("at"):
|
||||
out["at"] = a["at"]
|
||||
elif a.get("id"):
|
||||
out["id"] = a["id"]
|
||||
out["index"] = a.get("index", 0)
|
||||
return out
|
||||
|
||||
|
||||
def run_tool(name, a):
|
||||
"""(content list, is_error)."""
|
||||
if name == "gui_launch":
|
||||
class Args:
|
||||
pass
|
||||
args = Args()
|
||||
args.exe = a.get("exe")
|
||||
args.offscreen = bool(a.get("offscreen", True))
|
||||
args.port = None
|
||||
args.log = None
|
||||
args.wait = 60.0
|
||||
args.rest = list(a.get("args", []))
|
||||
# cmd_launch prints its own JSON and exits the process on failure, so
|
||||
# it is the argument shapes that are reused here, not the function.
|
||||
import io
|
||||
import contextlib
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
guictl.cmd_launch(args)
|
||||
return [{"type": "text", "text": buf.getvalue().strip()}], False
|
||||
|
||||
if name == "gui_screenshot":
|
||||
# Downscaled and JPEG by default: a 1600x950 PNG is 2 MB of base64,
|
||||
# which costs more context than the picture is worth.
|
||||
fmt = a.get("format", "jpg")
|
||||
img = guictl.call("/ui/screenshot",
|
||||
{"width": int(a.get("width", 1280)), "format": fmt,
|
||||
"quality": int(a.get("quality", 80))}, raw=True)
|
||||
content = [{"type": "image",
|
||||
"data": base64.b64encode(img).decode("ascii"),
|
||||
"mimeType": "image/png" if fmt == "png" else "image/jpeg"}]
|
||||
if a.get("path"):
|
||||
with open(a["path"], "wb") as f:
|
||||
f.write(img)
|
||||
content.append({"type": "text", "text": a["path"]})
|
||||
return content, False
|
||||
|
||||
routes = {
|
||||
"gui_state": ("/ui/state", lambda a: {}),
|
||||
"gui_tree": ("/ui/tree", lambda a: {
|
||||
k: v for k, v in (("q", a.get("q")), ("window", a.get("window")),
|
||||
("named", "0" if a.get("all") else "1"))
|
||||
if v is not None}),
|
||||
"gui_click": ("/ui/click", lambda a: dict(
|
||||
point_args(a), button=a.get("button", 0),
|
||||
double="1" if a.get("double") else "0")),
|
||||
"gui_move": ("/ui/move", point_args),
|
||||
"gui_drag": ("/ui/drag", lambda a: {
|
||||
"from": a["from"], "to": a["to"], "button": a.get("button", 0),
|
||||
"steps": a.get("steps", 8)}),
|
||||
"gui_scroll": ("/ui/scroll", lambda a: dict(
|
||||
point_args(a), dy=a.get("dy", -1.0))),
|
||||
"gui_key": ("/ui/key", lambda a: {"keys": a["keys"]}),
|
||||
"gui_text": ("/ui/text", lambda a: dict(
|
||||
point_args(a), value=a["value"],
|
||||
enter="1" if a.get("enter", True) else "0")),
|
||||
"gui_wait": ("/ui/wait", lambda a: {"frames": a.get("frames", 4)}),
|
||||
}
|
||||
if name not in routes:
|
||||
return [{"type": "text", "text": "unknown tool: " + name}], True
|
||||
path, build = routes[name]
|
||||
body = guictl.call(path, build(a))
|
||||
return [{"type": "text",
|
||||
"text": json.dumps(body, ensure_ascii=False, indent=2)}], False
|
||||
|
||||
|
||||
def handle(msg):
|
||||
"""The response to one request, or None for a notification."""
|
||||
method = msg.get("method")
|
||||
if method == "initialize":
|
||||
want = (msg.get("params") or {}).get("protocolVersion") or PROTOCOL
|
||||
return {"protocolVersion": want,
|
||||
"capabilities": {"tools": {}},
|
||||
"serverInfo": {"name": "spirula-gui", "version": "1"}}
|
||||
if method == "tools/list":
|
||||
return {"tools": TOOLS}
|
||||
if method == "tools/call":
|
||||
params = msg.get("params") or {}
|
||||
try:
|
||||
content, is_error = run_tool(params.get("name"),
|
||||
params.get("arguments") or {})
|
||||
except SystemExit as e:
|
||||
content, is_error = [{"type": "text", "text": str(e)}], True
|
||||
except Exception as e: # noqa: BLE001
|
||||
content, is_error = [{"type": "text",
|
||||
"text": "%s: %s" % (type(e).__name__, e)}], True
|
||||
return {"content": content, "isError": is_error}
|
||||
if method == "ping":
|
||||
return {}
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
out = sys.stdout
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if "id" not in msg:
|
||||
continue # a notification
|
||||
try:
|
||||
result = handle(msg)
|
||||
except Exception as e: # noqa: BLE001
|
||||
reply = {"jsonrpc": "2.0", "id": msg["id"],
|
||||
"error": {"code": -32603, "message": str(e)}}
|
||||
else:
|
||||
if result is None:
|
||||
reply = {"jsonrpc": "2.0", "id": msg["id"],
|
||||
"error": {"code": -32601,
|
||||
"message": "method not found: %s"
|
||||
% msg.get("method")}}
|
||||
else:
|
||||
reply = {"jsonrpc": "2.0", "id": msg["id"], "result": result}
|
||||
out.write(json.dumps(reply) + "\n")
|
||||
out.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+288
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drive the Spirula Studio GUI from a shell.
|
||||
|
||||
A thin client for the loopback control surface in src/app/gui/Automation.cpp,
|
||||
which the GUI opens when SS_GUI_AUTOMATION=1. Widgets are addressed by their
|
||||
i18n message name -- the part after "###" in an ImGui label -- so a script
|
||||
does not care which language the window is in.
|
||||
|
||||
python3 tools/guictl.py launch --offscreen
|
||||
python3 tools/guictl.py tree -q open
|
||||
python3 tools/guictl.py click open_dataset
|
||||
python3 tools/guictl.py shot /tmp/gui.png
|
||||
|
||||
Design and endpoint reference: docs/notes/gui-automation.md
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
def config_dir():
|
||||
base = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config")
|
||||
current = os.path.join(base, "spirula-studio")
|
||||
legacy = os.path.join(base, "spirulae-splat")
|
||||
if not os.path.isdir(current) and os.path.isdir(legacy):
|
||||
return legacy
|
||||
return current
|
||||
|
||||
|
||||
def endpoint():
|
||||
"""(base url, token), from the file the GUI writes when it starts."""
|
||||
host = os.environ.get("SS_GUI_AUTOMATION_HOST", "127.0.0.1")
|
||||
port = os.environ.get("SS_GUI_AUTOMATION_PORT")
|
||||
token = os.environ.get("SS_GUI_AUTOMATION_TOKEN", "")
|
||||
path = os.path.join(config_dir(), "automation.json")
|
||||
try:
|
||||
with open(path) as f:
|
||||
saved = json.load(f)
|
||||
port = port or str(saved.get("port", 7777))
|
||||
token = token or saved.get("token", "")
|
||||
except (OSError, ValueError):
|
||||
port = port or "7777"
|
||||
return "http://%s:%s" % (host, port), token
|
||||
|
||||
|
||||
def call(path, params=None, raw=False, timeout=30.0):
|
||||
base, token = endpoint()
|
||||
q = dict(params or {})
|
||||
if token:
|
||||
q["token"] = token
|
||||
url = "%s%s?%s" % (base, path, urllib.parse.urlencode(q))
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||
body = r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read()
|
||||
sys.stderr.write(body.decode("utf-8", "replace") + "\n")
|
||||
sys.exit(1)
|
||||
except urllib.error.URLError as e:
|
||||
sys.stderr.write("cannot reach %s: %s\n" % (base, e.reason))
|
||||
sys.stderr.write("is the GUI running with SS_GUI_AUTOMATION=1? "
|
||||
"(guictl.py launch)\n")
|
||||
sys.exit(2)
|
||||
if raw:
|
||||
return body
|
||||
return json.loads(body.decode("utf-8"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def target(args):
|
||||
"""id= / at= for the commands that act somewhere."""
|
||||
if getattr(args, "at", None):
|
||||
return {"at": args.at}
|
||||
return {"id": args.id, "index": args.index}
|
||||
|
||||
|
||||
def cmd_launch(args):
|
||||
exe = args.exe or os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "..", "build_cuda", "spirula")
|
||||
exe = os.path.abspath(exe)
|
||||
if not os.path.exists(exe):
|
||||
sys.exit("no such executable: %s" % exe)
|
||||
env = dict(os.environ)
|
||||
env["SS_GUI_AUTOMATION"] = "1"
|
||||
if args.port:
|
||||
env["SS_GUI_AUTOMATION_PORT"] = str(args.port)
|
||||
if args.offscreen:
|
||||
env["SS_GUI_OFFSCREEN"] = "1"
|
||||
log = open(args.log, "ab") if args.log else subprocess.DEVNULL
|
||||
rest = args.rest[1:] if args.rest[:1] == ["--"] else args.rest
|
||||
p = subprocess.Popen([exe] + rest, env=env, stdout=log, stderr=log,
|
||||
start_new_session=True)
|
||||
# The token file is written once the server is bound; polling /ui/state is
|
||||
# what actually says the first frame has been drawn.
|
||||
deadline = time.time() + args.wait
|
||||
while time.time() < deadline:
|
||||
if p.poll() is not None:
|
||||
sys.exit("the GUI exited with code %s" % p.returncode)
|
||||
try:
|
||||
base, token = endpoint()
|
||||
q = urllib.parse.urlencode({"token": token} if token else {})
|
||||
with urllib.request.urlopen("%s/ui/state?%s" % (base, q), timeout=1):
|
||||
print(json.dumps({"ok": True, "pid": p.pid}))
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(0.25)
|
||||
sys.exit("the GUI did not answer within %gs" % args.wait)
|
||||
|
||||
|
||||
def cmd_state(args):
|
||||
print(json.dumps(call("/ui/state"), indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def cmd_tree(args):
|
||||
params = {"named": "0" if args.all else "1"}
|
||||
if args.q:
|
||||
params["q"] = args.q
|
||||
if args.window:
|
||||
params["window"] = args.window
|
||||
items = call("/ui/tree", params)["items"]
|
||||
if args.json:
|
||||
print(json.dumps(items, indent=2, ensure_ascii=False))
|
||||
return
|
||||
for it in items:
|
||||
x0, y0, x1, y1 = it["rect"]
|
||||
print("%-38s %-28s %4.0f,%-4.0f %s"
|
||||
% (it["id"], it["label"][:28], (x0 + x1) / 2, (y0 + y1) / 2,
|
||||
it["window"]))
|
||||
print("(%d items)" % len(items), file=sys.stderr)
|
||||
|
||||
|
||||
def cmd_click(args):
|
||||
p = target(args)
|
||||
p["button"] = args.button
|
||||
p["double"] = "1" if args.double else "0"
|
||||
print(json.dumps(call("/ui/click", p)))
|
||||
|
||||
|
||||
def cmd_move(args):
|
||||
print(json.dumps(call("/ui/move", target(args))))
|
||||
|
||||
|
||||
def cmd_drag(args):
|
||||
print(json.dumps(call("/ui/drag", {"from": args.start, "to": args.end,
|
||||
"button": args.button,
|
||||
"steps": args.steps})))
|
||||
|
||||
|
||||
def cmd_scroll(args):
|
||||
p = target(args)
|
||||
p["dy"] = args.dy
|
||||
print(json.dumps(call("/ui/scroll", p)))
|
||||
|
||||
|
||||
def cmd_key(args):
|
||||
print(json.dumps(call("/ui/key", {"keys": args.keys})))
|
||||
|
||||
|
||||
def cmd_text(args):
|
||||
p = target(args)
|
||||
p["value"] = args.value
|
||||
p["enter"] = "0" if args.no_enter else "1"
|
||||
print(json.dumps(call("/ui/text", p)))
|
||||
|
||||
|
||||
def cmd_wait(args):
|
||||
print(json.dumps(call("/ui/wait", {"frames": args.frames})))
|
||||
|
||||
|
||||
def cmd_shot(args):
|
||||
params = {"format": args.format, "quality": args.quality}
|
||||
if args.width:
|
||||
params["width"] = args.width
|
||||
img = call("/ui/screenshot", params, raw=True)
|
||||
with open(args.out, "wb") as f:
|
||||
f.write(img)
|
||||
print(json.dumps({"ok": True, "path": args.out, "bytes": len(img)}))
|
||||
|
||||
|
||||
def cmd_script(args):
|
||||
"""One command per line: `click open_dataset`, `key Escape`, `wait 4`."""
|
||||
src = sys.stdin if args.file == "-" else open(args.file)
|
||||
for raw in src:
|
||||
line = raw.split("#", 1)[0].strip()
|
||||
if not line:
|
||||
continue
|
||||
print("+", line, file=sys.stderr)
|
||||
main(line.split())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def add_target(p):
|
||||
p.add_argument("id", nargs="?", help="widget id (the i18n message name)")
|
||||
p.add_argument("--at", help="x,y in ImGui display coordinates instead")
|
||||
p.add_argument("--index", type=int, default=0,
|
||||
help="which of several items sharing the id")
|
||||
|
||||
|
||||
def build_parser():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
p = sub.add_parser("launch", help="start the GUI with automation armed")
|
||||
p.add_argument("--exe", help="path to the spirula binary")
|
||||
p.add_argument("--offscreen", action="store_true")
|
||||
p.add_argument("--port", type=int)
|
||||
p.add_argument("--log", help="append the GUI's stdout/stderr here")
|
||||
p.add_argument("--wait", type=float, default=30.0)
|
||||
p.add_argument("rest", nargs=argparse.REMAINDER)
|
||||
p.set_defaults(func=cmd_launch)
|
||||
|
||||
p = sub.add_parser("state", help="screen, phase, queue depth, sizes")
|
||||
p.set_defaults(func=cmd_state)
|
||||
|
||||
p = sub.add_parser("tree", help="the widgets on screen")
|
||||
p.add_argument("-q", help="substring of the id or the visible label")
|
||||
p.add_argument("--window")
|
||||
p.add_argument("--all", action="store_true", help="include unnamed items")
|
||||
p.add_argument("--json", action="store_true")
|
||||
p.set_defaults(func=cmd_tree)
|
||||
|
||||
p = sub.add_parser("click")
|
||||
add_target(p)
|
||||
p.add_argument("--button", type=int, default=0, help="0 left 1 right 2 mid")
|
||||
p.add_argument("--double", action="store_true")
|
||||
p.set_defaults(func=cmd_click)
|
||||
|
||||
p = sub.add_parser("move", help="hover, e.g. to raise a tooltip")
|
||||
add_target(p)
|
||||
p.set_defaults(func=cmd_move)
|
||||
|
||||
p = sub.add_parser("drag", help="press, move, release -- viewport gestures")
|
||||
p.add_argument("start", help="x,y")
|
||||
p.add_argument("end", help="x,y")
|
||||
p.add_argument("--button", type=int, default=0)
|
||||
p.add_argument("--steps", type=int, default=8)
|
||||
p.set_defaults(func=cmd_drag)
|
||||
|
||||
p = sub.add_parser("scroll")
|
||||
add_target(p)
|
||||
p.add_argument("--dy", type=float, default=-1.0)
|
||||
p.set_defaults(func=cmd_scroll)
|
||||
|
||||
p = sub.add_parser("key", help='a chord, e.g. "Ctrl+S" or "Escape"')
|
||||
p.add_argument("keys")
|
||||
p.set_defaults(func=cmd_key)
|
||||
|
||||
p = sub.add_parser("text", help="focus an input and replace its contents")
|
||||
add_target(p)
|
||||
p.add_argument("value")
|
||||
p.add_argument("--no-enter", action="store_true")
|
||||
p.set_defaults(func=cmd_text)
|
||||
|
||||
p = sub.add_parser("wait", help="let N frames pass")
|
||||
p.add_argument("--frames", type=int, default=4)
|
||||
p.set_defaults(func=cmd_wait)
|
||||
|
||||
p = sub.add_parser("shot", help="write the framebuffer to an image file")
|
||||
p.add_argument("out")
|
||||
p.add_argument("--width", type=int, help="downscale to this width")
|
||||
p.add_argument("--format", choices=["png", "jpg"], default="png")
|
||||
p.add_argument("--quality", type=int, default=85, help="jpg only")
|
||||
p.set_defaults(func=cmd_shot)
|
||||
|
||||
p = sub.add_parser("script", help="run one command per line")
|
||||
p.add_argument("file", nargs="?", default="-")
|
||||
p.set_defaults(func=cmd_script)
|
||||
return ap
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = build_parser().parse_args(argv)
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user