fix(backup): include .octop and runtime dirs in workspace zip export

aglob("**/*") skips hidden paths, so downloads only packed a few root files.
Walk the local tree (and als/aglob fallbacks) so archives include the full workspace.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/octop/infra/backup/workspace_archive.py

@@ -4,10 +4,11 @@ from __future__ import annotations
 
 import asyncio
 import io
+import os
 import shutil
 import zipfile
-from pathlib import Path
-from typing import TYPE_CHECKING, Literal
+from pathlib import Path, PurePosixPath
+from typing import TYPE_CHECKING, Any, Literal
 
 from harness_agent.backends.utils import materialize_storage_path
 
@@ -18,6 +19,10 @@ WorkspaceImportMode = Literal["merge", "replace"]
 
 _SKIP_DIR_NAMES = frozenset({".git", "__pycache__", ".venv", "node_modules"})
 
+# ``aglob("**/*")`` skips dotfiles/dirs; these patterns recover hidden entries on
+# backends that only expose glob (no ``als``).
+_HIDDEN_GLOB_PATTERNS = (".**/**", "**/.*")
+
 
 def _safe_zip_name(name: str) -> str | None:
     raw = name.replace("\\", "/").strip().lstrip("/")
@@ -29,39 +34,116 @@ def _safe_zip_name(name: str) -> str | None:
     return "/".join(parts)
 
 
-async def _list_file_paths(workspace: BackendWorkspace) -> list[str]:
-    result = await workspace.aglob("**/*", ".")
-    if result is None:
+def _entry_rel_path(item: Any) -> tuple[str | None, bool]:
+    if isinstance(item, dict):
+        path = item.get("path")
+        is_dir = bool(item.get("is_dir"))
+    else:
+        path = getattr(item, "path", None)
+        is_dir = bool(getattr(item, "is_dir", False))
+    if not path:
+        return None, False
+    storage = str(path).replace("\\", "/")
+    return storage.lstrip("/") or None, is_dir
+
+
+def _normalize_listed_path(storage: str, workspace_dir: Path) -> str | None:
+    """Map a backend/listing path to a workspace-relative zip member name."""
+    raw = storage.replace("\\", "/")
+    ws_root = str(workspace_dir).replace("\\", "/").rstrip("/")
+    if raw == ws_root or raw.startswith(ws_root + "/"):
+        rel = raw[len(ws_root) :].lstrip("/")
+        return rel or None
+    return raw.lstrip("/") or None
+
+
+def _list_local_file_paths(workspace_dir: Path) -> list[str]:
+    """Walk the host workspace tree, including hidden dirs like ``.octop``."""
+    root = workspace_dir.resolve()
+    if not root.is_dir():
         return []
-    matches = getattr(result, "matches", None) or []
     paths: list[str] = []
-    for item in matches:
-        if isinstance(item, dict):
-            path = item.get("path")
-            is_dir = item.get("is_dir")
-        else:
-            path = getattr(item, "path", None)
-            is_dir = getattr(item, "is_dir", False)
-        if not path or is_dir:
-            continue
-        storage = str(path)
-        ws_root = str(workspace.workspace_dir)
-        if storage.startswith(ws_root):
-            rel = storage[len(ws_root) :].lstrip("/\\")
-            if rel:
-                paths.append(rel)
+    for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
+        dirnames[:] = [name for name in dirnames if name not in _SKIP_DIR_NAMES]
+        for name in filenames:
+            full = Path(dirpath) / name
+            if not full.is_file():
                 continue
-        paths.append(storage.lstrip("/"))
+            paths.append(full.relative_to(root).as_posix())
     return sorted(set(paths))
 
 
-def _clear_local_workspace(workspace_dir: Path) -> None:
-    """Clear local workspace content, keeping entries an archive cannot restore.
+async def _list_via_als(workspace: BackendWorkspace) -> list[str] | None:
+    """Recursively list files via ``als`` (includes hidden entries)."""
+    if getattr(workspace.backend, "als", None) is None:
+        return None
+    paths: list[str] = []
+    queue: list[str] = ["."]
+    seen_dirs: set[str] = set()
+    while queue:
+        current = queue.pop()
+        if current in seen_dirs:
+            continue
+        seen_dirs.add(current)
+        result = await workspace.als(current)
+        if result is None:
+            if current == ".":
+                return None
+            continue
+        for item in getattr(result, "entries", None) or []:
+            rel, is_dir = _entry_rel_path(item)
+            if rel is None:
+                continue
+            rel = _normalize_listed_path(rel, Path(workspace.workspace_dir))
+            if rel is None:
+                continue
+            name = PurePosixPath(rel).name
+            if name in _SKIP_DIR_NAMES:
+                continue
+            if is_dir:
+                queue.append(rel)
+            else:
+                paths.append(rel)
+    return sorted(set(paths))
 
-    The export glob never matches hidden paths, so harness system state (``.octop``
-    sessions / skills / auth, legacy ``.octop-auth``, ``.env``) is absent from every
-    archive. Deleting it here would destroy data the import can never put back, so
-    hidden entries are left untouched.
+
+async def _list_via_aglob(workspace: BackendWorkspace) -> list[str]:
+    """Fallback listing when there is no local mount and no ``als``."""
+    paths: set[str] = set()
+    for pattern in ("**/*", *_HIDDEN_GLOB_PATTERNS):
+        result = await workspace.aglob(pattern, ".")
+        if result is None:
+            continue
+        for item in getattr(result, "matches", None) or []:
+            rel, is_dir = _entry_rel_path(item)
+            if not rel or is_dir:
+                continue
+            normalized = _normalize_listed_path(rel, Path(workspace.workspace_dir))
+            if not normalized:
+                continue
+            if any(part in _SKIP_DIR_NAMES for part in PurePosixPath(normalized).parts):
+                continue
+            paths.add(normalized)
+    return sorted(paths)
+
+
+async def _list_file_paths(workspace: BackendWorkspace) -> list[str]:
+    """List every packable workspace file, including ``.octop`` / ``outbound`` / ``.env``."""
+    if _local_mount(workspace) is not None:
+        return await asyncio.to_thread(_list_local_file_paths, Path(workspace.workspace_dir))
+    via_als = await _list_via_als(workspace)
+    if via_als is not None:
+        return via_als
+    return await _list_via_aglob(workspace)
+
+
+def _clear_local_workspace(workspace_dir: Path) -> None:
+    """Clear visible local workspace content before a replace import.
+
+    Hidden entries (``.octop``, ``.env``, …) stay on disk: older archives may omit
+    them, and wiping system state that the zip never restores would destroy
+    sessions/auth. Files present in the archive under those paths are still
+    overwritten by the subsequent upload.
     """
     if not workspace_dir.is_dir():
         return

tests/unit/backup/test_workspace_archive.py

@@ -42,6 +42,47 @@ async def test_export_and_merge_import(tmp_path: Path) -> None:
     assert (tmp_path / "hello.txt").read_bytes() == b"hello"
 
 
+@pytest.mark.asyncio
+async def test_export_includes_hidden_and_runtime_dirs(tmp_path: Path) -> None:
+    """Workspace zip must pack the full tree, including ``.octop`` and ``outbound``.
+
+    ``aglob("**/*")`` skips dotfiles, which previously left archives with only a
+    handful of root markdown files.
+    """
+    home = tmp_path / "home"
+    workspace_dir = home / ".octop" / "workspaces" / "AGT1"
+    (workspace_dir / "outbound" / "shots").mkdir(parents=True)
+    (workspace_dir / "inbound").mkdir(parents=True)
+    (workspace_dir / ".octop" / "sessions").mkdir(parents=True)
+    (workspace_dir / ".octop" / "skills" / "demo").mkdir(parents=True)
+
+    (workspace_dir / "SOUL.md").write_bytes(b"soul")
+    (workspace_dir / ".env").write_bytes(b"KEY=1")
+    (workspace_dir / "outbound" / "shots" / "a.png").write_bytes(b"png")
+    (workspace_dir / "inbound" / "upload.txt").write_bytes(b"up")
+    (workspace_dir / ".octop" / "sessions" / "state.db").write_bytes(b"db")
+    (workspace_dir / ".octop" / "skills" / "demo" / "SKILL.md").write_bytes(b"skill")
+    (workspace_dir / ".git" / "objects").mkdir(parents=True)
+    (workspace_dir / ".git" / "objects" / "pack").write_bytes(b"skip-me")
+
+    backend = resolve_backend(
+        {"type": "filesystem", "root_dir": str(home), "virtual_mode": True},
+        workspace_dir=workspace_dir,
+    )
+    workspace = BackendWorkspace(backend, workspace_dir, system_files_path=".octop")
+
+    blob = await export_workspace_zip(workspace)
+    with zipfile.ZipFile(io.BytesIO(blob), "r") as zf:
+        names = set(zf.namelist())
+        assert zf.read("SOUL.md") == b"soul"
+        assert zf.read(".env") == b"KEY=1"
+        assert zf.read("outbound/shots/a.png") == b"png"
+        assert zf.read("inbound/upload.txt") == b"up"
+        assert zf.read(".octop/sessions/state.db") == b"db"
+        assert zf.read(".octop/skills/demo/SKILL.md") == b"skill"
+    assert ".git/objects/pack" not in names
+
+
 @pytest.mark.asyncio
 async def test_replace_clears_local_dir(tmp_path: Path) -> None:
     ws = tmp_path / "ws"
@@ -92,11 +133,10 @@ async def test_export_ignores_same_named_file_at_backend_root(tmp_path: Path) ->
 
 @pytest.mark.asyncio
 async def test_replace_import_keeps_system_state(tmp_path: Path) -> None:
-    """``replace`` must not delete state an archive can never carry.
+    """``replace`` must not wipe hidden state absent from older archives.
 
-    Hidden paths are outside the export glob, so ``.octop`` (sessions / skills /
-    auth) is absent from every archive; clearing it on replace destroyed data that
-    the import could not restore.
+    New exports include ``.octop``, but clearing it on replace would still destroy
+    sessions/auth when the uploaded zip is an older archive that omitted them.
     """
     ws = tmp_path / "ws"
     (ws / ".octop" / "sessions").mkdir(parents=True)
@@ -175,3 +215,25 @@ async def test_export_reads_remote_backend_over_stale_local_copy(tmp_path: Path)
     blob = await export_workspace_zip(workspace)
     with zipfile.ZipFile(io.BytesIO(blob), "r") as zf:
         assert zf.read("AGENTS.md") == b"remote AGENTS.md"
+
+
+@pytest.mark.asyncio
+async def test_export_mountless_includes_hidden_via_aglob(tmp_path: Path) -> None:
+    """Mountless backends without ``als`` still pack hidden paths via aglob fallback."""
+    workspace = BackendWorkspace(
+        _MountlessBackend(
+            {
+                "/SOUL.md": b"soul",
+                "/.octop/sessions/x.db": b"db",
+                "/outbound/a.txt": b"out",
+            }
+        ),
+        tmp_path,
+    )
+
+    blob = await export_workspace_zip(workspace)
+    with zipfile.ZipFile(io.BytesIO(blob), "r") as zf:
+        names = set(zf.namelist())
+        assert names == {"SOUL.md", ".octop/sessions/x.db", "outbound/a.txt"}
+        assert zf.read(".octop/sessions/x.db") == b"db"
+        assert zf.read("outbound/a.txt") == b"out"