Add documentation on how to measure model perplexity

countzero · Jul 3, 2023 · 4be5180 · 4be5180
1 parent ab52a52
commit 4be5180
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 5 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,3 @@
 
 # Ignore Sublime Text files.
 *.sublime-workspace
-
-# Ignore OpenBLAS directories and files
-vendor/OpenBLAS/*
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.0] - 2023-07-03
+
+### Added
+- Add dataset "wikitext-2-raw-v1"
+- Add documentation on how to measure model perplexity
+
 ## [1.0.0] - 2023-06-28
 
 ### Added

diff --git a/README.md b/README.md
@@ -103,6 +103,20 @@ You can now chat with the model:
     --interactive
 ```
 
+### Measure model perplexity
+
+Execute the following to measure the perplexity of the GGML formatted model:
+
+```PowerShell
+./vendor/llama.cpp/build/bin/Release/perplexity `
+    --model "./vendor/llama.cpp/models/open-llama-7B-open-instruct.ggmlv3.q4_K_M.bin" `
+    --ctx-size 2048 `
+    --n-predict 2048 `
+    --threads 16 `
+    --n-gpu-layers 32 `
+    --file "./vendor/wikitext-2-raw-v1/wikitext-2-raw/wiki.test.raw"
+```
+
 ### Rebuild llama.cpp
 
 Every time there is a new release of [llama.cpp](https://github.com/ggerganov/llama.cpp) you can simply execute the script to automatically rebuild everything:

diff --git a/rebuild_llama.cpp.ps1 b/rebuild_llama.cpp.ps1
@@ -37,6 +37,18 @@ if (-not(Test-Path -Path "./vendor/OpenBLAS/OpenBLAS-${openBLASVersion}-x64.zip"
         -Force
 }
 
+if (-not(Test-Path -Path "./vendor/wikitext-2-raw-v1/wikitext-2-raw-v1.zip")) {
+
+    Invoke-WebRequest `
+        -Uri "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip" `
+        -OutFile "./vendor/wikitext-2-raw-v1/wikitext-2-raw-v1.zip"
+
+    Expand-Archive `
+        -Path "./vendor/wikitext-2-raw-v1/wikitext-2-raw-v1.zip" `
+        -DestinationPath "./vendor/wikitext-2-raw-v1" `
+        -Force
+}
+
 function Resolve-UnixPath {
     Param ([String] $path)
     Write-Output ((Resolve-Path "$path").Path -replace '\\','/')
@@ -67,7 +79,7 @@ if (!(Select-String -Path "./vendor/llama.cpp/CMakeLists.txt" -Pattern $lines[0]
     Set-Content "./vendor/llama.cpp/CMakeLists.txt"
 }
 
-Remove-Item  -Path "./vendor/llama.cpp/build" -Force -Recurse
+Remove-Item -Path "./vendor/llama.cpp/build" -Force -Recurse
 
 New-Item -Path "./vendor/llama.cpp" -Name "build" -ItemType "directory"
 

diff --git a/vendor/llama.cpp b/vendor/llama.cpp
diff --git a/vendor/wikitext-2-raw-v1/.gitignore b/vendor/wikitext-2-raw-v1/.gitignore
@@ -0,0 +1,3 @@
+# Ignore everything in this directory except this file.
+*
+!.gitignore
+0 −5		CMakeLists.txt
+1 −0		README.md
+36 −5		convert.py
+1 −1		examples/common.cpp
+3 −2		examples/common.h
+6 −3		examples/embd-input/embd-input-lib.cpp
+1 −3		examples/embd-input/embd-input.h
+2 −2		examples/embedding/embedding.cpp
+1 −1		examples/main/README.md
+2 −2		examples/main/main.cpp
+2 −2		examples/perplexity/perplexity.cpp
+1 −1		examples/server/README.md
+120 −30		examples/server/server.cpp
+5 −4		examples/train-text-from-scratch/train-text-from-scratch.cpp
+46 −17		ggml-cuda.cu
+0 −4		ggml-cuda.h
+3 −1		ggml-metal.m
+53 −9		ggml.c
+3 −0		ggml.h
+57 −12		llama.cpp
+8 −6		llama.h