# READING NOTES (added before publish) # - This is the raw session log: ANSI color codes and progress-bar output are left as-is. # - In the first "E3 locations" section, every verify timing reads "verify-failed" — that was a # timing-capture bug in my harness (output redirected away), not a verify failure. The "E3b" section # re-runs the diagnosis and shows `hf cache verify` passing. Ignore the first E3 timings entirely. # - Sections that feed the article tables: # env-var matrix table <- "E2: env var combination matrix" # benchmark, dd+tokenizer<- "E3c: N=5 benchmark" (individual runs) # benchmark, model load <- "E3d: full model load" # benchmark, load+forward<- "E3e: load + first forward pass" # cleanup workflow <- "E4"/"E4b" # revision dedup <- "E1b" (inside E3c section) # - vhdx sizes (the "still 5.11 GB" table) were measured host-side with PowerShell # (logical size + allocated size via GetCompressedFileSizeW); they are summarized in the article. # HF cache experiments - session logs (2026-08-06) # Host: Windows 11 build 26200 / WSL 2.7.3.0 / Ubuntu 26.04 / hub 1.26.0 / transformers 5.14.1 # Single Samsung NVMe (MZVL2512): C: and ext4.vhdx on the same physical disk. AC power, balanced plan. ################ E2: env var combination matrix ################ === E2 matrix run 2026-08-05T17:36:18Z | hub Version: 1.26.0 | transformers Version: 5.14.1 === combo=1_none | tool=hf | landing=/home/ljk5998/.cache/huggingface/hub combo=1_none | tool=transformers | landing=/home/ljk5998/.cache/huggingface/hub combo=2_HF_HOME | tool=hf | landing=/home/ljk5998/hftest/home1/hub combo=2_HF_HOME | tool=transformers | landing=/home/ljk5998/hftest/home1/hub combo=3_HOME+HUB_CACHE | tool=hf | landing=/home/ljk5998/hftest/hubcache combo=3_HOME+HUB_CACHE | tool=transformers | landing=/home/ljk5998/hftest/hubcache combo=4_HUB_CACHE_only | tool=hf | landing=/home/ljk5998/hftest/hubcache combo=4_HUB_CACHE_only | tool=transformers | landing=/home/ljk5998/hftest/hubcache combo=5_HOME+TRANSFORMERS_CACHE | tool=hf | landing=/home/ljk5998/hftest/home1/hub combo=5_HOME+TRANSFORMERS_CACHE | tool=transformers | landing=/home/ljk5998/hftest/home1/hub combo=6_HUB_CACHE_vs_legacy | tool=hf | landing=/home/ljk5998/hftest/hubcache combo=6_HUB_CACHE_vs_legacy | tool=transformers | landing=/home/ljk5998/hftest/hubcache --- warnings combo5 (TRANSFORMERS_CACHE, transformers v5) --- Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. --- warnings combo6 (HUGGINGFACE_HUB_CACHE legacy) --- Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. --- gen2 removal proof --- Usage: hf cache [OPTIONS] COMMAND [ARGS]... Try 'hf cache --help' for help. Error: No such command 'scan'. Available commands for 'hf cache': list List cached repositories or revisions. prune Remove detached revisions and incomplete downloads from the cache. rm Remove cached repositories or revisions. verify Verify checksums for a single repo revision from cache or a local directory. Run 'hf cache --help' for full details. (exit 2) --- gen3 current --- Error: Cache directory not found: /home/ljk5998/.cache/huggingface/hub Hint: set HF_DEBUG=1 as environment variable for full traceback. E2 DONE ################ E1/E3/E4: structure, locations, cleanup ################ === E1 structure run 2026-08-05T17:40:12Z === --- download (network) --- Fetching 10 files: 0%| | 0/10 [00:00 ../../blobs/6634c8cc3133b3848ec74b9f275acaaa1ea618ab lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:40 README.md -> ../../blobs/4b8373851d093eb9f3017443f27781c6971eff24 lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:40 config.json -> ../../blobs/0dbb161213629a23f0fc00ef286e6b1e366d180f lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:40 generation_config.json -> ../../blobs/dfc11073787daf1b0f9c0f1499487ab5f4c93738 lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:40 merges.txt -> ../../blobs/20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0 lrwxrwxrwx 1 ljk5998 ljk5998 76 Aug 6 02:40 model.safetensors -> ../../blobs/fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:40 tokenizer.json -> ../../blobs/443909a61d429dff23010e5bddd28ff530edda00 lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:40 tokenizer_config.json -> ../../blobs/07bfe0640cb5a0037f9322287fbfc682806cf672 lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:40 vocab.json -> ../../blobs/4783fe10ac3adce15ac8f358ef5462739852c569 --- du: 링크 크기 vs 실제 (겉보기 이중계산 데모) --- blobs: 954M snapshots(du -sh, 링크): 12K snapshots(du -shL, 역참조): 954M repo total(du -sh): 954M --- transformers 재사용 (dedup: 재다운로드 없어야 함) --- [transformers] PyTorch was not found. Models won't be available and only tokenizers, configuration and file/data utilities can be used. Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. tokenizer_load_wall=3.93 s blobs before=10 after=10 (같으면 dedup 작동) --- .no_exist 데모 --- /home/ljk5998/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/.no_exist /home/ljk5998/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/.no_exist/7ae557604adf67be50417f59c2c2f167def9a775 /home/ljk5998/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/.no_exist/7ae557604adf67be50417f59c2c2f167def9a775/special_tokens_map.json /home/ljk5998/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/.no_exist/7ae557604adf67be50417f59c2c2f167def9a775/chat_template.jinja /home/ljk5998/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/.no_exist/7ae557604adf67be50417f59c2c2f167def9a775/added_tokens.json E1 DONE === E3 locations run 2026-08-05T17:40:32Z === --- 복사 준비 (동일 캐시 3벌) --- copy_to_mnt_c_wall=13.48 s 복사 완료: ext4=954M mnt_c=1.9G --- bench A_ext4 (/home/ljk5998/e3/ext4hub) --- A_ext4 run1 verify_wall=verify-faileds A_ext4 run2 verify_wall=verify-faileds A_ext4 run3 verify_wall=verify-faileds A_ext4 run4 verify_wall=verify-faileds A_ext4 run5 verify_wall=verify-faileds A_ext4 raw_read_wall=s (943M) --- bench B_symlink (/home/ljk5998/e3/linkhub) --- B_symlink run1 verify_wall=verify-faileds B_symlink run2 verify_wall=verify-faileds B_symlink run3 verify_wall=verify-faileds B_symlink run4 verify_wall=verify-faileds B_symlink run5 verify_wall=verify-faileds B_symlink raw_read_wall=s (943M) --- bench C_mnt_c_9p (/mnt/c/hfcache-test/hub) --- C_mnt_c_9p run1 verify_wall=verify-faileds C_mnt_c_9p run2 verify_wall=verify-faileds C_mnt_c_9p run3 verify_wall=verify-faileds C_mnt_c_9p run4 verify_wall=verify-faileds C_mnt_c_9p run5 verify_wall=verify-faileds C_mnt_c_9p raw_read_wall=s (943M) --- 심링크 경로로 hf cache ls 동작 확인 --- ID SIZE LAST_ACCESSED LAST_MODIFIED REFS -------------------------------- ------ -------------- ------------- ---- model/Qwen/Qwen2.5-0.5B-Instruct 999.6M 49 seconds ago 1 minute ago main  Found 1 repo(s) for a total of 1 revision(s) and 999.6M on disk. E3 DONE === E4 cleanup run 2026-08-05T17:41:52Z === --- hf cache ls (현행 출력) --- ID SIZE LAST_ACCESSED LAST_MODIFIED REFS -------------------------------- ------ ----------------- ----------------- ---- model/Qwen/Qwen2.5-0.5B-Instruct 999.6M 1 minute ago 1 minute ago main model/hf-internal-testing/tin... 12.5M a few seconds ago a few seconds ago main Hint: Use `--no-truncate` or `--format json` to display full values.  Found 2 repo(s) for a total of 2 revision(s) and 1.0G on disk. --- du before --- 966M /home/ljk5998/.cache/huggingface/hub --- hf cache rm (저장소 단위 삭제) --- Warning: Could not find in cache: - hf-internal-testing/tiny-random-gpt2 Nothing to delete. --- hf cache prune --- No unreferenced revisions or incomplete downloads found. Nothing to prune. --- du after --- 966M /home/ljk5998/.cache/huggingface/hub --- legacy CLI 경고 원문 (huggingface-cli) --- Warning: `huggingface-cli` is deprecated and no longer works. Use `hf` instead.  Hint: Install `hf`: Standalone (recommended): curl -LsSf https://hf.co/cli/install.sh | bash Using Homebrew: brew install hf Using pip: pip install huggingface_hub === v4 대조 셀 (transformers<5에서 TRANSFORMERS_CACHE 동작 확인) === v4 venv transformers=4.57.6 --- v4: HF_HOME+TRANSFORMERS_CACHE 조합 (v5와 대조) --- /home/ljk5998/hfenv4/lib/python3.14/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. warnings.warn( v4 landing: TRANSFORMERS_CACHE=HIT | HF_HOME=empty E4 DONE === E3b bench fix run 2026-08-05T17:43:58Z === --- verify 진단 (ext4) --- ✓ Verified 10 file(s) for model 'Qwen/Qwen2.5-0.5B-Instruct'. All checksums match. repo_id: Qwen/Qwen2.5-0.5B-Instruct repo_type: model checked: 10 path: /home/ljk5998/e3/ext4hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775 --- bench A_ext4 --- A_ext4 run1 raw_read=0.63s A_ext4 run2 raw_read=0.60s A_ext4 run3 raw_read=0.55s A_ext4 run1 tokenizer_load=2.54s A_ext4 run2 tokenizer_load=2.61s A_ext4 run3 tokenizer_load=2.49s --- bench B_symlink --- B_symlink run1 raw_read=0.91s B_symlink run2 raw_read=0.56s B_symlink run3 raw_read=0.58s B_symlink run1 tokenizer_load=2.54s B_symlink run2 tokenizer_load=2.54s B_symlink run3 tokenizer_load=2.45s --- bench C_mnt_c_9p --- C_mnt_c_9p run1 raw_read=4.80s C_mnt_c_9p run2 raw_read=4.70s C_mnt_c_9p run3 raw_read=4.78s C_mnt_c_9p run1 tokenizer_load=2.86s C_mnt_c_9p run2 tokenizer_load=2.84s C_mnt_c_9p run3 tokenizer_load=2.76s --- /mnt/c 심링크 보존 복사 테스트 (cp -r, 역참조 없이) --- 크기: 966M 스냅샷 파일 상태 (심링크가 살아남았나?): total 0 lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:44 LICENSE -> ../../blobs/6634c8cc3133b3848ec74b9f275acaaa1ea618ab lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:44 README.md -> ../../blobs/4b8373851d093eb9f3017443f27781c6971eff24 lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:44 config.json -> ../../blobs/0dbb161213629a23f0fc00ef286e6b1e366d180f lrwxrwxrwx 1 ljk5998 ljk5998 52 Aug 6 02:44 generation_config.json -> ../../blobs/dfc11073787daf1b0f9c0f1499487ab5f4c93738 심링크 보존 캐시로 오프라인 로드 시도: [transformers] PyTorch was not found. Models won't be available and only tokenizers, configuration and file/data utilities can be used. LOAD OK E3b DONE === E4b rm fix run 2026-08-05T17:44:54Z === --- hf cache rm --help (문법 확인) --- Usage: hf cache rm [OPTIONS] TARGETS... Remove cached repositories or revisions. Arguments: TARGETS... One or more repo IDs (e.g. model/bert-base-uncased), repo-level hf:// URIs, or revision hashes to delete. [required] Options: --cache-dir TEXT Cache directory to scan (defaults to Hugging Face cache). -y, --yes Skip confirmation prompt. --dry-run / --no-dry-run Preview deletions without removing anything. [default: no-dry-run] -h, --help Show this message and exit. Formatting options: --format [auto|human|agent|json|quiet] Output format. Defaults to 'auto' which picks 'agent' or 'human' based on the --- du before --- 966M /home/ljk5998/.cache/huggingface/hub --- rm 시도 1: model/ 접두 --- About to delete 1 repo(s) totalling 12.5M. - model/hf-internal-testing/tiny-random-gpt2 (entire repo) Delete repo: /home/ljk5998/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-gpt2 Cache deletion done. Saved 12.5M. ✓ Deleted 1 repo(s) and 1 revision(s); freed 12.5M. repos_deleted: 1 revisions_deleted: 1 freed: 12.5M --- du after --- 954M /home/ljk5998/.cache/huggingface/hub tiny 저장소 잔존 여부: 0 E4b DONE ################ E3c: N=5 benchmark + revision dedup ################ === E3c N=5 rerun 2026-08-06T00:22:53Z === --- 측정 조건 (WSL 측) --- C:\134 /mnt/c 9p rw,noatime,aname=drvfs;path=C:\;uid=1000;gid=1000;symlinkroot=/mnt/,cache=5,access=client,msize=65536,trans=fd,rfd=5,wfd=5 0 0 6.6.114.1-microsoft-standard-WSL2 재다운로드 완료: 954M 사본: ext4=954M mnt_c=954M (cp -a 보존 복사) --- bench A_ext4 (N=5, 개별값) --- A_ext4 raw_read run1 = 0.60s A_ext4 raw_read run2 = 0.64s A_ext4 raw_read run3 = 0.59s A_ext4 raw_read run4 = 0.68s A_ext4 raw_read run5 = 0.65s A_ext4 tokenizer run1 = 2.85s A_ext4 tokenizer run2 = 2.64s A_ext4 tokenizer run3 = 2.81s A_ext4 tokenizer run4 = 2.67s A_ext4 tokenizer run5 = 2.68s --- bench B_symlink (N=5, 개별값) --- B_symlink raw_read run1 = 1.01s B_symlink raw_read run2 = 0.58s B_symlink raw_read run3 = 0.85s B_symlink raw_read run4 = 0.58s B_symlink raw_read run5 = 0.84s B_symlink tokenizer run1 = 2.74s B_symlink tokenizer run2 = 2.68s B_symlink tokenizer run3 = 2.74s B_symlink tokenizer run4 = 2.81s B_symlink tokenizer run5 = 2.93s --- bench C_mnt_c (N=5, 개별값) --- C_mnt_c raw_read run1 = 4.99s C_mnt_c raw_read run2 = 5.11s C_mnt_c raw_read run3 = 4.90s C_mnt_c raw_read run4 = 4.95s C_mnt_c raw_read run5 = 4.79s C_mnt_c tokenizer run1 = 3.30s C_mnt_c tokenizer run2 = 3.32s C_mnt_c tokenizer run3 = 3.24s C_mnt_c tokenizer run4 = 3.43s C_mnt_c tokenizer run5 = 3.18s === E1b: revision 간 blob dedup 검증 === 총 커밋 수: 9 revision A(최신): 71034c5d8bde / B(직전): 914a3832e3d8 Fetching 10 files: 0%| | 0/10 [00:00