mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(relevance): keep an empty HF cache variable, as huggingface_hub does
huggingface_hub reads its cache variables with os.getenv(name, default), so a variable that is set but empty still counts: HF_HUB_CACHE='' is the working directory, HF_HOME='' is `hub` and XDG_CACHE_HOME='' is `huggingface/hub` under it. The Rust resolver treated an empty value as unset and fell through to the next variable, so with HF_HUB_CACHE='' and HF_HOME set it looked up and loaded a snapshot from HF_HOME/hub, a cache Python never reads. - hf_hub_cache_dir() reads every path-valued variable as set when it is present, empty or not; expansion and precedence are unchanged. - tests/parity/fixtures/hf_hub_cache/cases.json gains the empty cases for HF_HUB_CACHE, HUGGINGFACE_HUB_CACHE, HF_HOME and XDG_CACHE_HOME, checked against huggingface_hub.constants in Python and against the port in Rust.
This commit is contained in:
@@ -6,9 +6,8 @@
|
||||
//! All network-free: [`hub_cache_dir`] picks the one cache directory Python
|
||||
//! would use, [`snapshot_path`] resolves a repo at a revision inside it,
|
||||
//! [`hf_token`] finds the access token and [`hub_api`] builds a client from
|
||||
//! them. One known divergence: an empty variable counts as unset here, where
|
||||
//! Python would take it as a path relative to the working directory (see
|
||||
//! [`hub_cache_dir`]). The pinned revisions themselves live
|
||||
//! them. A path variable that is set but empty counts as set, as in Python
|
||||
//! (see [`hub_cache_dir`]). The pinned revisions themselves live
|
||||
//! next to each loader and are kept equal to `_PINNED_REVISIONS` in
|
||||
//! `headroom/onnx_runtime.py` by tests on the Python side.
|
||||
//!
|
||||
@@ -74,8 +73,10 @@ pub(crate) fn snapshot_path(cache: &Path, repo: &str, revision: &str) -> PathBuf
|
||||
/// a `~` that only appears once a variable is expanded stays literal, as it
|
||||
/// does in Python. The home directory comes from `std::env::home_dir`, which
|
||||
/// falls back to the user database when `HOME` is unset, as `expanduser`
|
||||
/// does. An empty variable counts as unset; Python would treat it as a path
|
||||
/// relative to the working directory. `tests/parity/fixtures/hf_hub_cache/`
|
||||
/// does. A variable that is set but empty still counts, as with `os.getenv`:
|
||||
/// `HF_HUB_CACHE=''` is the working directory itself, `HF_HOME=''` is `hub`
|
||||
/// and `XDG_CACHE_HOME=''` is `huggingface/hub` under it, and none of them
|
||||
/// falls through to a lower-priority cache. `tests/parity/fixtures/hf_hub_cache/`
|
||||
/// holds the cases both languages are checked against.
|
||||
pub(crate) fn hub_cache_dir() -> Option<PathBuf> {
|
||||
let hub = match env_path("HF_HUB_CACHE").or_else(|| env_path("HUGGINGFACE_HUB_CACHE")) {
|
||||
@@ -219,11 +220,10 @@ fn sends_in_clear_text(endpoint: &str) -> bool {
|
||||
!loopback
|
||||
}
|
||||
|
||||
/// A path-valued variable; an empty one counts as unset.
|
||||
/// A path-valued variable. An empty one is still set, as `os.getenv` reads
|
||||
/// it, so it never falls through to the next variable.
|
||||
fn env_path(name: &str) -> Option<PathBuf> {
|
||||
std::env::var_os(name)
|
||||
.filter(|v| !v.is_empty())
|
||||
.map(PathBuf::from)
|
||||
std::env::var_os(name).map(PathBuf::from)
|
||||
}
|
||||
|
||||
/// `expandvars(expanduser(path))`, the order `huggingface_hub.constants` uses.
|
||||
@@ -472,10 +472,38 @@ pub(crate) mod tests {
|
||||
Some(p("home").join(".cache").join("huggingface").join("hub"))
|
||||
);
|
||||
|
||||
// An empty variable counts as unset, and a leading `~` expands.
|
||||
std::env::set_var("HF_HUB_CACHE", "");
|
||||
// A leading `~` expands.
|
||||
std::env::set_var("HF_HOME", "~/models");
|
||||
assert_eq!(hub_cache_dir(), Some(p("home").join("models").join("hub")));
|
||||
|
||||
// A set but empty variable is the working directory, as in Python,
|
||||
// and still wins over every variable after it.
|
||||
std::env::set_var("HF_HUB_CACHE", "");
|
||||
assert_eq!(hub_cache_dir(), Some(PathBuf::new()));
|
||||
std::env::remove_var("HF_HUB_CACHE");
|
||||
std::env::set_var("HF_HOME", "");
|
||||
std::env::set_var("XDG_CACHE_HOME", p("xdg"));
|
||||
assert_eq!(hub_cache_dir(), Some(PathBuf::from("hub")));
|
||||
}
|
||||
|
||||
/// An empty `HF_TOKEN_PATH` names no readable file, so there is no token;
|
||||
/// it must not fall back to `HF_HOME/token`. Python's `get_token()` raises
|
||||
/// `IsADirectoryError` here (it reads `.`), so this case is Rust-only
|
||||
/// rather than in the shared fixture.
|
||||
#[test]
|
||||
fn an_empty_hf_token_path_does_not_fall_back_to_hf_home() {
|
||||
let _guard = crate::test_support::env_lock();
|
||||
let _env = CacheEnv::cleared_with(
|
||||
["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_TOKEN_PATH"].map(String::from),
|
||||
);
|
||||
let hf_home = tempfile::tempdir().unwrap();
|
||||
std::fs::write(hf_home.path().join("token"), "hf_from_home\n").unwrap();
|
||||
std::env::set_var("HF_HOME", hf_home.path());
|
||||
|
||||
std::env::remove_var("HF_TOKEN_PATH");
|
||||
assert_eq!(hf_token().as_deref(), Some("hf_from_home"));
|
||||
std::env::set_var("HF_TOKEN_PATH", "");
|
||||
assert_eq!(hf_token(), None);
|
||||
}
|
||||
|
||||
/// Cases checked against Python itself (`posixpath`, `ntpath` and
|
||||
|
||||
@@ -233,6 +233,45 @@
|
||||
"nt": "{root}/hub"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "an empty HF_HUB_CACHE is the working directory, not HF_HOME (the #4438 review case)",
|
||||
"env": {
|
||||
"HF_HUB_CACHE": "",
|
||||
"HF_HOME": "{root}/fallback-home"
|
||||
},
|
||||
"expected": ""
|
||||
},
|
||||
{
|
||||
"name": "an empty legacy HUGGINGFACE_HUB_CACHE is the working directory",
|
||||
"env": {
|
||||
"HUGGINGFACE_HUB_CACHE": "",
|
||||
"HF_HOME": "{root}/fallback-home"
|
||||
},
|
||||
"expected": ""
|
||||
},
|
||||
{
|
||||
"name": "an empty HF_HUB_CACHE still wins over a set legacy HUGGINGFACE_HUB_CACHE",
|
||||
"env": {
|
||||
"HF_HUB_CACHE": "",
|
||||
"HUGGINGFACE_HUB_CACHE": "{root}/legacy"
|
||||
},
|
||||
"expected": ""
|
||||
},
|
||||
{
|
||||
"name": "an empty HF_HOME is hub under the working directory, not XDG_CACHE_HOME",
|
||||
"env": {
|
||||
"HF_HOME": "",
|
||||
"XDG_CACHE_HOME": "{root}/xdg"
|
||||
},
|
||||
"expected": "hub"
|
||||
},
|
||||
{
|
||||
"name": "an empty XDG_CACHE_HOME is huggingface/hub under the working directory, not the home default",
|
||||
"env": {
|
||||
"XDG_CACHE_HOME": ""
|
||||
},
|
||||
"expected": "huggingface/hub"
|
||||
},
|
||||
{
|
||||
"name": "nothing set: the default under the home directory",
|
||||
"env": {},
|
||||
|
||||
Reference in New Issue
Block a user