fix(pm): download npm-hosted tools through the user's npm registry
pm/lock.json and the Npm/AgentBrowser templates record registry.npmjs.org, and pinned_source handed that URL straight to the downloader, so ~/.npmrc / npm_config_registry were ignored and closed networks could not provision npm or agent-browser. pinned_source now routes public-npm URLs through the configured registry (npm's own precedence: npm_config_registry, then the user npmrc); the lock's SHA256 still verifies the bytes and progress keeps the lockfile URL. npm_dist_tags reads from the same registry. The E2E cell test_npm_registry_mirror_serves_pm_npm_download drops its gate. Co-authored-by: funky-xamarin <30426178+Wenfengcheng@users.noreply.github.com>
This commit is contained in:
committed by
Teknium
co-authored by
funky-xamarin
parent
86f20a367a
commit
a31e618e70
@@ -2,12 +2,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from pm.downloader import Source
|
||||
from pm.index_config import npm_registry_url
|
||||
|
||||
_LAYOUT = json.loads(Path(__file__).with_name("artifact-mirror.json").read_text(encoding="utf-8"))
|
||||
_LAYOUT = json.loads(Path(__file__).with_name("artifact-mirror.json").read_text(encoding="utf-8-sig"))
|
||||
KEY_PREFIX = _LAYOUT["prefix"]
|
||||
PUBLIC_PREFIX = _LAYOUT["origin"] + "/" + KEY_PREFIX
|
||||
|
||||
@@ -25,4 +27,6 @@ def mirror_url(sha256: str) -> str:
|
||||
|
||||
def pinned_source(url: str, dest: Path, sha256: str) -> Source:
|
||||
archive = mirror_url(sha256)
|
||||
# The lock records registry.npmjs.org; a user's npm mirror serves the same pinned bytes (#123132).
|
||||
url = npm_registry_url(url, os.environ)
|
||||
return Source(url, dest, sha256, fallbacks=() if url == archive else (archive,))
|
||||
|
||||
@@ -90,6 +90,34 @@ def pip_conf_index_url(env: Mapping[str, str]) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
NPM_PUBLIC_REGISTRY = "https://registry.npmjs.org/"
|
||||
|
||||
|
||||
def npm_registry(env: Mapping[str, str]) -> str:
|
||||
"""The npm registry the user configured, as npm resolves it: ``npm_config_registry`` (any
|
||||
case) beats ``registry=`` in the user npmrc (``npm_config_userconfig`` or ``~/.npmrc``)."""
|
||||
lowered = {key.lower(): value for key, value in env.items()}
|
||||
value = (lowered.get("npm_config_registry") or "").strip()
|
||||
if not value:
|
||||
npmrc = Path(lowered.get("npm_config_userconfig") or Path.home() / ".npmrc").expanduser()
|
||||
try:
|
||||
lines = npmrc.read_text(encoding="utf-8-sig").splitlines()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
lines = []
|
||||
for line in lines:
|
||||
key, sep, candidate = line.partition("=")
|
||||
if sep and key.strip().lower() == "registry":
|
||||
value = candidate.strip().strip("\"'")
|
||||
return value.rstrip("/") + "/" if value.startswith(("https://", "http://")) else NPM_PUBLIC_REGISTRY
|
||||
|
||||
|
||||
def npm_registry_url(url: str, env: Mapping[str, str]) -> str:
|
||||
"""*url* on the configured npm registry; non-npm URLs are returned unchanged."""
|
||||
if not url.startswith(NPM_PUBLIC_REGISTRY):
|
||||
return url
|
||||
return npm_registry(env) + url[len(NPM_PUBLIC_REGISTRY):]
|
||||
|
||||
|
||||
def bridged_index_settings(ambient: Mapping[str, str]) -> dict[str, str]:
|
||||
"""The uv index/transport settings *ambient* asks for, pip knobs translated.
|
||||
|
||||
|
||||
+2
-1
@@ -386,7 +386,8 @@ class Store:
|
||||
destination = scratch / entry_name / url.rsplit("/", 1)[-1]
|
||||
sources.append(pinned_source(url, destination, digest))
|
||||
|
||||
urls = {str(source.dest): source.url for source in sources}
|
||||
# Progress keeps the lockfile URL even when the transport is a mirror (#123132).
|
||||
urls = {str(source.dest): artifact["url"] for source, artifact in zip(sources, artifacts)}
|
||||
|
||||
def tick(done, total, ranges):
|
||||
if progress is not None:
|
||||
|
||||
+3
-1
@@ -353,7 +353,9 @@ def github_release_tags(repo: str, *, strip_prefix: str = "") -> list[str]:
|
||||
|
||||
|
||||
def npm_dist_tags(name: str) -> dict:
|
||||
return _get_json(f"https://registry.npmjs.org/-/package/{name}/dist-tags")
|
||||
from pm.index_config import npm_registry
|
||||
|
||||
return _get_json(f"{npm_registry(os.environ)}-/package/{name}/dist-tags")
|
||||
|
||||
|
||||
def node_latest_versions() -> list[str]:
|
||||
|
||||
@@ -11,7 +11,7 @@ pre-fetched at fixture time and verified against the pins: the PM runtime's lock
|
||||
|
||||
Classes: an update that ships a PM runtime change restages it with ``uv sync --locked`` against
|
||||
the committed lock while pip's mirror is bridged into uv (#124418, #123943, #122112), and PM's
|
||||
npm-hosted tool downloads (#123132: hardcoded registry.npmjs.org).
|
||||
npm-hosted tool downloads (#123132: the configured npm registry serves the pinned tarball).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -105,14 +105,9 @@ def test_npm_registry_mirror_serves_pm_npm_download(inst):
|
||||
finally:
|
||||
edge.close()
|
||||
public = _public_index_hits(edge)
|
||||
with known_failure(
|
||||
r"reached the public registry \['refused registry\.npmjs\.org",
|
||||
"gated on #123132: PM downloads npm-hosted tools from a hardcoded registry.npmjs.org "
|
||||
"URL and ignores the configured npm registry",
|
||||
):
|
||||
assert not public and r.rc == 0, (
|
||||
f"npm provisioning with a registry mirror failed or reached the public registry {public}\n"
|
||||
+ r.report(inst))
|
||||
assert not public and r.rc == 0, (
|
||||
f"npm provisioning with a registry mirror failed or reached the public registry {public}\n"
|
||||
+ r.report(inst))
|
||||
assert (inst.sb.hermes_home / "tools" / tool["entry"]).is_dir(), "npm was not re-provisioned\n" + r.report(inst)
|
||||
assert any(h.path.endswith(tarball) and h.status == 200 for h in mirror.hits), (
|
||||
"the tarball was not served by the mirror\n" + r.report(inst))
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import hashlib
|
||||
import importlib
|
||||
import io
|
||||
import os
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
@@ -53,3 +54,21 @@ def test_cold_consumers_recover_after_upstream_removal(tmp_path, dl_server, monk
|
||||
result = engine.stage_only(package.name, "linux-arm64-bionic")
|
||||
assert (result / "tool.txt").read_bytes() == b"pinned and preserved"
|
||||
assert any(path == "/archive/" + digest for path, *_ in RangeHandler.ranges_seen)
|
||||
|
||||
|
||||
def test_npm_artifacts_follow_the_users_npm_registry(tmp_path, monkeypatch):
|
||||
"""#123132: lock URLs name registry.npmjs.org; ~/.npmrc or npm_config_registry picks the mirror."""
|
||||
from pm.artifact_mirror import pinned_source
|
||||
|
||||
digest = "a" * 64
|
||||
lock_url = "https://registry.npmjs.org/npm/-/npm-10.9.2.tgz"
|
||||
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
|
||||
for key in [k for k in os.environ if k.lower().startswith("npm_config_")]:
|
||||
monkeypatch.delenv(key)
|
||||
assert pinned_source(lock_url, tmp_path / "npm.tgz", digest).url == lock_url
|
||||
(tmp_path / ".npmrc").write_text("; corp\nregistry = https://npm.corp.example/npm/\n", encoding="utf-8")
|
||||
assert pinned_source(lock_url, tmp_path / "npm.tgz", digest).url == "https://npm.corp.example/npm/npm/-/npm-10.9.2.tgz"
|
||||
monkeypatch.setenv("NPM_CONFIG_REGISTRY", "https://env.corp.example")
|
||||
assert pinned_source(lock_url, tmp_path / "npm.tgz", digest).url == "https://env.corp.example/npm/-/npm-10.9.2.tgz"
|
||||
other = "https://github.com/x/y/releases/download/v1/y.tgz"
|
||||
assert pinned_source(other, tmp_path / "y.tgz", digest).url == other
|
||||
|
||||
Reference in New Issue
Block a user