mirror of
https://github.com/rohitg00/ai-engineering-from-scratch.git
synced 2026-10-02 01:54:39 +08:00
fix(phase-10/10): perplexity was different on every run (#358)
The simulated log-probs are seeded from the text so the same text scores the same way, and STEP 3 compares Strong/Medium/Weak on that basis. hash() of a str is salted per interpreter process, so the seed changed every run - three consecutive runs gave Strong-model perplexity 1.20, 1.16, 1.13. Seed from hashlib.sha256 instead. Three runs now produce byte-identical output, and the Strong < Medium < Weak ordering the lesson relies on is preserved (1.13 / 1.44 / 2.52). Co-authored-by: thejesh23 <thejesh23@users.noreply.github.com>
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
|
||||
@@ -110,7 +111,7 @@ def perplexity(log_probs):
|
||||
|
||||
|
||||
def token_log_probs_simulated(text, model_quality=0.8):
|
||||
np.random.seed(hash(text) % 2**31)
|
||||
np.random.seed(int(hashlib.sha256(text.encode()).hexdigest()[:8], 16) % 2**31)
|
||||
tokens = text.split()
|
||||
log_probs = []
|
||||
for i, token in enumerate(tokens):
|
||||
@@ -188,7 +189,7 @@ def demo_model_bad(prompt):
|
||||
|
||||
|
||||
def demo_model_random(prompt):
|
||||
np.random.seed(hash(prompt) % 2**31)
|
||||
np.random.seed(int(hashlib.sha256(prompt.encode()).hexdigest()[:8], 16) % 2**31)
|
||||
words = ["yes", "no", "maybe", "42", "Paris", "unknown", "error"]
|
||||
return words[np.random.randint(len(words))]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user