support initializing PPISP exposure from EXIF

2026-10-02 02:44:54 +08:00 · 2026-08-21 20:23:24 -04:00
parent bcc1736639
commit 784cd1db9e
11 changed files with 173 additions and 10 deletions
@@ -10,6 +10,7 @@
 #include "i18n/catalog/Log.h"
 #include "data/CameraMath.h"
 #include "data/Knn.h"
+#include "sfm/core/Exif.h"

 #ifndef _WIN32
 #include <ftw.h>
@@ -639,6 +640,44 @@ static void check_cuda_runtime() {
 }
 #endif  // SS_BACKEND_VULKAN

+// PPISP exposure seeds: mean-relative EXIF EV x 0.5 per POST-split slot; empty
+// when no image has the tags. The 0.5: PPISP multiplies the sRGB-encoded
+// render, where a bracketed +1 EV measures x2^0.49 (0.34-0.76 by tone curve).
+static std::vector<float> exif_exposure_evs(const ParsedDataset& ds,
+                                            const PostSplitCameras& post,
+                                            int& n_found) {
+    int64_t n = ds.num_cameras;
+    std::vector<double> ev(n, 0.0);
+    std::vector<char> has(n, 0);
+    double sum = 0.0;
+    n_found = 0;
+    for (int64_t i = 0; i < n; i++) {
+        double v;
+        if (sfm::exifExposureEv(sfm::readExif(ds.image_filenames[i]), v)) {
+            ev[i] = v;
+            has[i] = 1;
+            sum += v;
+            // sum += std::exp2(v);
+            n_found++;
+        }
+    }
+    if (n_found == 0) return {};
+    double mean = sum / n_found;
+    // double mean = std::log2(sum / n_found);
+    std::vector<float> out((size_t)post.n_post, 0.0f);
+    for (int64_t i = 0; i < n; i++) {
+        if (!has[i]) continue;
+        float v = 0.5f * (float)(ev[i] - mean);
+        if (post.K_per_camera.empty()) {
+            out[i] = v;
+        } else {
+            for (int k = 0; k < post.K_per_camera[i]; k++)
+                out[post.post_offsets[i] + k] = v;
+        }
+    }
+    return out;
+}
+
 void TrainerSession::setup_engine() {
 #ifndef SS_BACKEND_VULKAN
    check_cuda_runtime();
@@ -790,7 +829,16 @@ void TrainerSession::setup_engine() {
    }
    if (cfg.use_ppisp &&
        (cfg.use_adagrad_ppisp_optim ? cfg.ppisp_adagrad_lr : cfg.ppisp_lr) > 0.0f) {
-        engine_init_ppisp(n_grids, cfg.ppisp_param_type, cfg.use_adagrad_ppisp_optim);
+        std::vector<float> exif_ev;
+        if (cfg.ppisp_exposure_from_exif) {
+            int n_exif = 0;
+            exif_ev = exif_exposure_evs(ds, post, n_exif);
+            if (n_exif > 0)
+                log(lfmt(lmsg::ppisp_exif_exposure,
+                         {(long long)n_exif, (long long)ds.num_cameras}));
+        }
+        engine_init_ppisp(n_grids, cfg.ppisp_param_type,
+                          cfg.use_adagrad_ppisp_optim, exif_ev);
        st.ppisp_init = true;
    }

@@ -243,6 +243,7 @@ inline int train_tier_rank(const char* tier) {
    X(bool, use_adagrad_bilagrid_optim, true, "correction", "advanced", "")  \
    X(bool, use_ppisp, true, "correction", "basic", "")                      \
    X(std::string, ppisp_param_type, "no_crf", "correction", "basic", "original|rqs|no_crf") \
+    X(bool, ppisp_exposure_from_exif, false, "correction", "basic", "")      \
    X(bool, apply_ppisp_before_bilagrid, true, "correction", "advanced", "") \
    X(bool, use_adagrad_ppisp_optim, true, "correction", "advanced", "")     \
    X(float, ppisp_reg_exposure_mean, 1.0f, "correction", "advanced", "")    \
@@ -217,14 +217,10 @@ void engine_bilagrid_forward(TorchTensorView cam_indices);
 void engine_bilagrid_optim_step(int step, const BilagridStepConfig& cfg);

 // --- PPISP (RGB only, applied AFTER bilagrid). ---
-// Allocates a per-camera PPISP parameter table and seeds it with the type's
-// default values ("original" -> 12 zeros, 12 zeros, 3x(a,a,b,0); "rqs" -> all 0).
-//
-// use_adagrad: if true, PPISP uses unscheduled AdaGrad (lr_decay=0,
-// weight_decay=0, initial_accumulator_value=0, eps=1e-15) instead of Adam.
-// No quantization path is provided for PPISP either way -- the parameter
-// table is small (N_cam x ~36 floats), so the fp32 store/state suffices.
-void engine_init_ppisp(int n_grids, std::string param_type, bool use_adagrad);
+// Table seeded with the param_type's defaults; exposure_init optionally seeds
+// params[:, 0] with [n_grids] log2 gains. use_adagrad: AdaGrad over Adam.
+void engine_init_ppisp(int n_grids, std::string param_type, bool use_adagrad,
+                       const std::vector<float>& exposure_init = {});

 // Apply PPISP forward in place on the current rendered RGB; saves a pre-PPISP
 // copy used by backward. cam_indices: [C_batch] int32, or null/empty for
@@ -38,9 +38,13 @@ static TorchTensorView _ppisp_cam_indices_tv() {
    return TorchTensorView((uint64_t)ptr, 4, {n});
 }

-void engine_init_ppisp(int n_grids, std::string param_type, bool use_adagrad) {
+void engine_init_ppisp(int n_grids, std::string param_type, bool use_adagrad,
+                       const std::vector<float>& exposure_init) {
    if (n_grids <= 0)
        throw std::runtime_error("engine_init_ppisp: n_grids must be > 0");
+    if (!exposure_init.empty() && (int)exposure_init.size() != n_grids)
+        throw std::runtime_error(
+            "engine_init_ppisp: exposure_init size must be n_grids");
    int P;
    if (param_type == "original" || param_type == "") P = 36;
    else if (param_type == "rqs") P = 39;
@@ -59,6 +63,12 @@ void engine_init_ppisp(int n_grids, std::string param_type, bool use_adagrad) {
    } else {
        engine().ppisp.params.zero();
    }
+    // Exposure is params[:, 0] in every layout (shaders/ppisp.slang). Same
+    // stream as the default init, so the seeds land after it.
+    float* params = engine().ppisp.params.data_ptr();
+    for (size_t i = 0; i < exposure_init.size(); i++)
+        backend::memcpy_async(params + i * P, &exposure_init[i], sizeof(float),
+                              backend::MemcpyKind::HostToDevice, kPpispStream);
    engine().ppisp.enabled = true;
    engine().ppisp.optim_initialized = false;
 }
@@ -918,6 +918,21 @@ SS_MSG(ray_depth_straight_ahead,
    RU("расстояние прямо вперёд"),
    TR("ileri doğru mesafe"));

+SS_MSG(ppisp_exif_exposure,
+    EN("PPISP exposure initialized from EXIF ({0} of {1} photos)"),
+    JA("PPISP の露出を EXIF から初期化しました（{1} 枚中 {0} 枚）"),
+    ZH_HANS("已从 EXIF 初始化 PPISP 曝光（{1} 张照片中的 {0} 张）"),
+    ZH_HANT("已從 EXIF 初始化 PPISP 曝光（{1} 張照片中的 {0} 張）"),
+    KO("EXIF에서 PPISP 노출을 초기화했습니다({1}장 중 {0}장)"),
+    DE("PPISP-Belichtung aus EXIF initialisiert ({0} von {1} Fotos)"),
+    FR("Exposition PPISP initialisée depuis l'EXIF ({0} photos sur {1})"),
+    ES("Exposición PPISP inicializada desde EXIF ({0} de {1} fotos)"),
+    PT("Exposição PPISP inicializada a partir do EXIF ({0} de {1} fotos)"),
+    IT("Esposizione PPISP inizializzata dall'EXIF ({0} foto su {1})"),
+    NL("PPISP-belichting geïnitialiseerd uit EXIF ({0} van {1} foto's)"),
+    RU("Экспозиция PPISP инициализирована из EXIF ({0} из {1} фото)"),
+    TR("PPISP pozlaması EXIF'ten başlatıldı ({1} fotoğraftan {0})"));
+
 // Printed when the input images are EXRs and no colour space was given on the
 // command line; {0} is the gamut read out of the file.
 SS_MSG(exr_color_space,
@@ -6513,6 +6513,65 @@ SS_MSG(ppisp_param_type_help,
       "bir ton eğrisi ekler. `rqs` ise karanlık bölgelerde daha iyi davranan "
       "bir ton eğrisi kullanır."));

+SS_MSG(ppisp_exposure_from_exif,
+    EN("Exposure init from EXIF"), JA("EXIF による露出の初期化"),
+    ZH_HANS("用 EXIF 初始化曝光"), ZH_HANT("用 EXIF 初始化曝光"),
+    KO("EXIF로 노출 초기화"), DE("Belichtungsstart aus EXIF"),
+    FR("Exposition initiale depuis l'EXIF"), ES("Exposición inicial desde EXIF"),
+    PT("Exposição inicial do EXIF"), IT("Esposizione iniziale da EXIF"),
+    NL("Belichting starten vanuit EXIF"), RU("Начальная экспозиция из EXIF"),
+    TR("EXIF'ten pozlama başlangıcı"));
+SS_MSG(ppisp_exposure_from_exif_help,
+    EN("Seed each photo's exposure correction from the shutter, aperture and "
+       "ISO its EXIF records, relative to the set's average, instead of from "
+       "zero. Photos without those tags start at the average. Helps when "
+       "exposure varies across the capture."),
+    JA("各写真の露出補正を、ゼロからではなく EXIF に記録されたシャッター速度・"
+       "絞り・ISO から、セットの平均を基準に初期化します。タグのない写真は平均"
+       "から始まります。撮影中に露出が変わる場合に有効です。"),
+    ZH_HANS("用 EXIF 记录的快门、光圈和 ISO（相对于整组的平均值）来初始化每张"
+            "照片的曝光校正，而不是从零开始。没有这些标签的照片从平均值开始。"
+            "在拍摄过程中曝光有变化时有帮助。"),
+    ZH_HANT("用 EXIF 記錄的快門、光圈和 ISO（相對於整組的平均值）來初始化每張"
+            "照片的曝光校正，而不是從零開始。沒有這些標籤的照片從平均值開始。"
+            "在拍攝過程中曝光有變化時有幫助。"),
+    KO("각 사진의 노출 보정을 0이 아니라 EXIF에 기록된 셔터·조리개·ISO에서, 전"
+       "체 평균을 기준으로 초기화합니다. 태그가 없는 사진은 평균에서 시작합니"
+       "다. 촬영 중 노출이 달라질 때 도움이 됩니다."),
+    DE("Die Belichtungskorrektur jedes Fotos aus Verschlusszeit, Blende und ISO "
+       "seiner EXIF-Daten starten, relativ zum Durchschnitt des Satzes, statt "
+       "bei null. Fotos ohne diese Tags starten beim Durchschnitt. Hilft, wenn "
+       "die Belichtung über die Aufnahme variiert."),
+    FR("Amorcer la correction d'exposition de chaque photo à partir de la vitesse, "
+       "de l'ouverture et de l'ISO enregistrés dans son EXIF, par rapport à la "
+       "moyenne de l'ensemble, plutôt qu'à zéro. Les photos sans ces balises "
+       "partent de la moyenne. Utile quand l'exposition varie au fil de la prise "
+       "de vue."),
+    ES("Inicializar la corrección de exposición de cada foto a partir del obturador, "
+       "la apertura y el ISO que registra su EXIF, relativos a la media del conjunto, "
+       "en lugar de desde cero. Las fotos sin esas etiquetas parten de la media. "
+       "Ayuda cuando la exposición varía a lo largo de la captura."),
+    PT("Inicializar a correção de exposição de cada foto a partir do obturador, "
+       "da abertura e do ISO registrados no EXIF, relativos à média do conjunto, "
+       "em vez de partir do zero. Fotos sem essas tags partem da média. Ajuda "
+       "quando a exposição varia ao longo da captura."),
+    IT("Inizializzare la correzione di esposizione di ogni foto da otturatore, "
+       "apertura e ISO registrati nel suo EXIF, relativi alla media del set, invece "
+       "che da zero. Le foto senza quei tag partono dalla media. Aiuta quando "
+       "l'esposizione varia durante la ripresa."),
+    NL("De belichtingscorrectie van elke foto starten vanuit de sluitertijd, het "
+       "diafragma en de ISO uit de EXIF, relatief aan het gemiddelde van de set, "
+       "in plaats van vanaf nul. Foto's zonder die tags starten op het gemiddelde. "
+       "Helpt wanneer de belichting tijdens de opname varieert."),
+    RU("Инициализировать коррекцию экспозиции каждого фото по выдержке, диафрагме "
+       "и ISO из его EXIF относительно среднего по набору, а не с нуля. Фото без "
+       "этих тегов начинают со среднего. Помогает, когда экспозиция меняется по "
+       "ходу съёмки."),
+    TR("Her fotoğrafın pozlama düzeltmesini sıfırdan değil, EXIF'inde kayıtlı "
+       "enstantane, diyafram ve ISO'dan, kümenin ortalamasına göre başlatır. Bu "
+       "etiketleri olmayan fotoğraflar ortalamadan başlar. Pozlama çekim boyunca "
+       "değişiyorsa yardımcı olur."));
+
 SS_MSG(apply_ppisp_before_bilagrid,
    EN("Camera correction first"), JA("カメラ補正を先に適用"),
    ZH_HANS("先做相机校正"), ZH_HANT("先做相機校正"),
@@ -27,6 +27,7 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
+#include <limits>
 #include <string>
 #include <vector>

@@ -41,6 +42,14 @@ struct ExifData {
    int focal_plane_unit = 0;       // Exif:FocalPlaneResolutionUnit (2=in, 3=cm, 4=mm, 5=um)
    int pixel_width = 0, pixel_height = 0;  // Exif:PixelXDimension/PixelYDimension

+    double exposure_time = 0;   // Exif:ExposureTime, seconds
+    double f_number = 0;        // Exif:FNumber
+    double iso = 0;             // Exif:PhotographicSensitivity (+ SOS/REI/ISOSpeed fallbacks)
+    // APEX forms, NaN when absent (Tv=0 means a 1 s exposure, so 0 cannot
+    // mark "missing"): t = 2^-Tv, N = 2^(Av/2).
+    double shutter_apex  = std::numeric_limits<double>::quiet_NaN();  // Exif:ShutterSpeedValue
+    double aperture_apex = std::numeric_limits<double>::quiet_NaN();  // Exif:ApertureValue
+
    bool hasFocal() const { return focal_35mm > 0 || focal_mm > 0; }
 };

@@ -134,6 +143,14 @@ inline void parseIfd(const TiffReader& r, size_t off, ExifData& out, int depth)
            case 0x8769:  // Exif sub-IFD, where the lens tags live
                if (tiffValue(r, e, v) && v > 0) parseIfd(r, (size_t)v, out, depth + 1);
                break;
+            case 0x829A: if (tiffValue(r, e, v) && v > 0) out.exposure_time = v; break;
+            case 0x829D: if (tiffValue(r, e, v) && v > 0) out.f_number = v; break;
+            case 0x8827: if (tiffValue(r, e, v) && v > 0) out.iso = v; break;
+            case 0x8831: case 0x8832: case 0x8833:  // SOS / REI / ISOSpeed
+                if (out.iso == 0 && tiffValue(r, e, v) && v > 0) out.iso = v;
+                break;
+            case 0x9201: if (tiffValue(r, e, v)) out.shutter_apex = v; break;
+            case 0x9202: if (tiffValue(r, e, v)) out.aperture_apex = v; break;
            case 0x920A: if (tiffValue(r, e, v)) out.focal_mm = v; break;
            case 0xA405: if (tiffValue(r, e, v)) out.focal_35mm = v; break;
            case 0xA20E: if (tiffValue(r, e, v)) out.focal_plane_x_res = v; break;
@@ -239,6 +256,23 @@ inline double exifFocalPx(const ExifData& e, int width, int height) {
    return 0;
 }

+// Relative capture exposure in EV stops, log2(t / N^2 * ISO), preferring the
+// direct tags over their APEX forms. A missing component counts as 1; false
+// only when the file records none of the three.
+inline bool exifExposureEv(const ExifData& e, double& ev) {
+    if (!e.valid) return false;
+    double t = e.exposure_time > 0 ? e.exposure_time
+             : std::isfinite(e.shutter_apex) ? std::exp2(-e.shutter_apex) : 0;
+    double N = e.f_number > 0 ? e.f_number
+             : std::isfinite(e.aperture_apex) ? std::exp2(e.aperture_apex / 2) : 0;
+    double s = e.iso;
+    if (t <= 0 && N <= 0 && s <= 0) return false;
+    double rel = (t > 0 ? t : 1) / ((N > 0 ? N : 1) * (N > 0 ? N : 1)) * (s > 0 ? s : 1);
+    if (!(rel > 0) || !std::isfinite(rel)) return false;
+    ev = std::log2(rel);
+    return true;
+}
+
 // The identity two images must share to be assumed the same physical camera:
 // make, model and frame size. Empty when EXIF cannot identify the camera, which
 // means "do not group by it".