| import os | |
| import torch | |
| DATA_DIR = "data" | |
| TOKENIZER_PATH = "tokenizer.json" | |
| CHECKPOINT_DIR = "checkpoints" | |
| TRAIN_TOKENS_CACHE = "data/train_tokens.npy" | |
| VAL_TOKENS_CACHE = "data/val_tokens.npy" | |
| DEVICE = "cuda" if torch.cuda.is_available() else "cpu" | |
| DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32 | |
| def load_dotenv(path=".env"): | |
| """Tiny dependency-free .env loader. Populates os.environ (does not overwrite | |
| variables already set in the real environment). Lets HF_TOKEN live in a | |
| gitignored .env file instead of being hardcoded or pasted into the shell.""" | |
| if not os.path.exists(path): | |
| return | |
| with open(path, encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if not line or line.startswith("#") or "=" not in line: | |
| continue | |
| key, _, value = line.partition("=") | |
| key, value = key.strip(), value.strip().strip('"').strip("'") | |
| os.environ.setdefault(key, value) | |
| load_dotenv() | |
| # HuggingFace token (optional for public datasets like wikitext-103; used if present). | |
| HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") | |