diff --git a/tamingllms/_build/.doctrees/environment.pickle b/tamingllms/_build/.doctrees/environment.pickle index a82719a..45e0882 100644 Binary files a/tamingllms/_build/.doctrees/environment.pickle and b/tamingllms/_build/.doctrees/environment.pickle differ diff --git a/tamingllms/_build/.doctrees/notebooks/structured_output.doctree b/tamingllms/_build/.doctrees/notebooks/structured_output.doctree index 2438965..9fb3b7c 100644 Binary files a/tamingllms/_build/.doctrees/notebooks/structured_output.doctree and b/tamingllms/_build/.doctrees/notebooks/structured_output.doctree differ diff --git a/tamingllms/_build/html/_sources/notebooks/structured_output.ipynb b/tamingllms/_build/html/_sources/notebooks/structured_output.ipynb index b4dd536..fbade00 100644 --- a/tamingllms/_build/html/_sources/notebooks/structured_output.ipynb +++ b/tamingllms/_build/html/_sources/notebooks/structured_output.ipynb @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -145,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -160,11 +160,11 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - "prompt = f\"\"\"\n", + "prompt = \"\"\"\n", "Generate a two-person discussion about the key financial data from the following text in JSON format.\n", "\n", "\n", @@ -179,35 +179,38 @@ " }\n", "}\n", "\n", - "\n", - "TEXT: {sec_filing}\n", "\"\"\"\n", "\n", "response = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[{\"role\": \"user\", \"content\": prompt}]\n", + " model=\"gpt-4o-mini\",\n", + " messages=[\n", + " {\"role\": \"system\", \"content\": prompt},\n", + " {\"role\": \"user\", \"content\": sec_filing}\n", + " ]\n", ")" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "```json\n", "{\n", " \"Person1\": {\n", " \"name\": \"Alice\",\n", - " \"statement\": \"The revenue for Q1 has increased by 20% compared to last year.\"\n", + " \"statement\": \"The aggregate market value of Apple's stock held by non-affiliates is approximately $2.63 trillion.\"\n", " },\n", " \"Person2\": {\n", " \"name\": \"Bob\",\n", - " \"statement\": \"That's great news! What about the net profit margin?\"\n", + " \"statement\": \"That's impressive! I also noticed that they have around 15.1 billion shares of common stock outstanding.\"\n", " }\n", - "}\n" + "}\n", + "```\n" ] } ], @@ -218,7 +221,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -234,16 +237,16 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "True" + "False" ] }, - "execution_count": 33, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -256,7 +259,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We observe the LLM did provide a response in JSON format. However, it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format." + "We observe the LLM provided a response in JSON format. However, it was enclosed by json markdown tags! While further prompt engineering could lead to a pure JSON response it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format." ] }, { diff --git a/tamingllms/_build/html/notebooks/structured_output.html b/tamingllms/_build/html/notebooks/structured_output.html index f17167c..c160747 100644 --- a/tamingllms/_build/html/notebooks/structured_output.html +++ b/tamingllms/_build/html/notebooks/structured_output.html @@ -409,28 +409,29 @@

-
prompt = f"""
+
prompt = """
 Generate a two-person discussion about the key financial data from the following text in JSON format.
 
 <JSON_FORMAT>
-{
-   "Person1": {
-     "name": "Alice",
-     "statement": "The revenue for Q1 has increased by 20% compared to last year."
-   },
-   "Person2": {
-     "name": "Bob",
+{
+   "Person1": {
+     "name": "Alice",
+     "statement": "The revenue for Q1 has increased by 20% compared to last year."
+   },
+   "Person2": {
+     "name": "Bob",
      "statement": "That's great news! What about the net profit margin?"
-   }
-}
+   }
+}
 </JSON_FORMAT>
-
-TEXT: {sec_filing}
 """
 
 response = client.chat.completions.create(
-    model="gpt-3.5-turbo",
-    messages=[{"role": "user", "content": prompt}]
+    model="gpt-4o-mini",
+    messages=[
+        {"role": "system", "content": prompt},
+        {"role": "user", "content": sec_filing}
+    ]
 )
 

 

@@ -444,16 +445,18 @@ 

-
{
+
```json
+{
    "Person1": {
      "name": "Alice",
-     "statement": "The revenue for Q1 has increased by 20% compared to last year."
+     "statement": "The aggregate market value of Apple's stock held by non-affiliates is approximately $2.63 trillion."
    },
    "Person2": {
      "name": "Bob",
-     "statement": "That's great news! What about the net profit margin?"
+     "statement": "That's impressive! I also noticed that they have around 15.1 billion shares of common stock outstanding."
    }
 }
+```
 

 

 

@@ -479,12 +482,12 @@ 

-
True
+
False
 

 

 

 

-
We observe the LLM did provide a response in JSON format. However, it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format.

+
We observe the LLM provided a response in JSON format. However, it was enclosed by json markdown tags! While further prompt engineering could lead to a pure JSON response it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format.

 
 

 
4.3.2. JSON Mode (Fine-Tuned)ΒΆ

diff --git a/tamingllms/_build/html/searchindex.js b/tamingllms/_build/html/searchindex.js
index af262ae..67a153f 100644
--- a/tamingllms/_build/html/searchindex.js
+++ b/tamingllms/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["markdown/intro", "markdown/preface", "markdown/toc", "notebooks/alignment", "notebooks/cost", "notebooks/evals", "notebooks/input", "notebooks/local", "notebooks/safety", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/preface.md", "markdown/toc.md", "notebooks/alignment.ipynb", "notebooks/cost.ipynb", "notebooks/evals.ipynb", "notebooks/input.ipynb", "notebooks/local.ipynb", "notebooks/safety.ipynb", "notebooks/structured_output.ipynb"], "titles": ["2. About the Book", "1. Preface", "Taming LLMs", "7. Preference-Based Alignment", "9. The Falling Cost Paradox", "3. The Evals Gap", "5. Managing Input Data", "8. Local LLMs in Practice", "6. Safety", "4. Structured Output"], "terms": {"am": [0, 8], "alwai": [0, 3, 4, 5, 6, 9], "do": [0, 3, 4, 5, 6, 7, 8, 9], "which": [0, 3, 4, 5, 6, 7, 8, 9], "cannot": [0, 3, 4, 5, 7, 8], "order": [0, 3, 5, 6, 8, 9], "mai": [0, 1, 3, 4, 5, 6, 7, 8, 9], "learn": [0, 3, 5, 6, 7, 8, 9], "how": [0, 1, 3, 4, 5, 6, 7, 8, 9], "pablo": [0, 5], "picasso": 0, "In": [0, 3, 4, 5, 6, 7, 8, 9], "recent": [0, 3, 4, 5, 6, 7, 8, 9], "year": [0, 2, 3, 4, 5, 6, 7, 8, 9], "larg": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "languag": [0, 1, 2, 4, 5, 6, 7, 8, 9], "model": [0, 1, 2, 4, 6, 8, 9], "llm": [0, 1, 3, 6, 9], "have": [0, 1, 3, 4, 5, 6, 7, 8, 9], "emerg": [0, 3, 4, 6, 7, 8, 9], "transform": [0, 1, 3, 5, 6, 7, 8, 9], "forc": [0, 5, 9], "technologi": [0, 1, 4, 5, 6, 7, 8], "promis": [0, 3, 4, 5, 8], "revolution": [0, 8], "build": [0, 2, 3, 5, 6, 7, 8, 9], "product": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "interact": [0, 3, 4, 5, 6, 7, 8, 9], "comput": [0, 3, 4, 5, 6, 7, 8, 9], "from": [0, 1, 4, 5, 6, 7, 8, 9], "chatgpt": [0, 3, 4, 7, 9], "llama": [0, 3, 4, 5, 8, 9], "github": [0, 2, 3, 4, 5, 7, 8, 9], "copilot": 0, "claud": [0, 3, 5, 7, 8], "artifact": 0, "system": [0, 3, 4, 5, 6, 7, 8, 9], "captur": [0, 1, 3, 5, 7, 8], "public": [0, 3, 5, 7, 8], "imagin": [0, 7], "spark": 0, "gold": [0, 3, 6, 8], "rush": 0, "ai": [0, 3, 4, 5, 7, 9], "power": [0, 2, 3, 4, 5, 7, 8, 9], "applic": [0, 1, 2, 3, 4, 6, 7, 8, 9], "howev": [0, 3, 4, 5, 6, 7, 8, 9], "beneath": 0, "surfac": [0, 5], "technolog": [0, 1, 4, 5, 6, 8], "revolut": [0, 4], "li": [0, 3, 5, 7, 8, 9], "complex": [0, 1, 3, 5, 6, 7, 8, 9], "landscap": [0, 3, 5, 7], "softwar": [0, 1, 3, 4, 6, 7, 8, 9], "develop": [0, 1, 3, 4, 5, 6, 7, 8, 9], "tech": [0, 7, 8], "leader": [0, 2, 5, 8], "must": [0, 3, 4, 5, 7, 8, 9], "navig": [0, 2, 5, 7, 8], "focus": [0, 3, 4, 5, 6, 7, 8, 9], "bring": [0, 3, 7], "awar": [0, 3, 4, 5, 6, 8], "limit": [0, 1, 2, 4, 5, 6, 7, 8, 9], "har": [0, 2, 5], "solut": [0, 2, 4, 5, 6, 7, 8], "overcom": [0, 5, 6], "them": [0, 1, 3, 4, 5, 6, 7, 8, 9], "robust": [0, 3, 4, 5, 6, 7, 8, 9], "It": [0, 3, 4, 5, 6, 7, 8, 9], "offer": [0, 3, 4, 5, 6, 7, 8, 9], "critic": [0, 2, 3, 4, 5, 6, 7, 8, 9], "implement": [0, 2, 3, 4, 5, 7, 9], "back": [0, 5, 6, 7, 8, 9], "reproduc": [0, 1, 2, 5, 7], "exampl": [0, 1, 2, 3, 5, 7, 8, 9], "while": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "mani": [0, 1, 3, 4, 5, 6, 7, 8, 9], "resourc": [0, 3, 4, 5, 6, 7, 8], "cover": [0, 3, 4, 5, 6, 7, 8, 9], "capabl": [0, 1, 2, 4, 5, 6, 7, 8, 9], "specif": [0, 3, 4, 5, 6, 7, 9], "hidden": [0, 3, 8], "pitfal": [0, 1, 3, 4, 5, 6, 7, 9], "engin": [0, 1, 2, 3, 4, 5, 7, 8], "technic": [0, 1, 2, 3, 5, 7, 9], "face": [0, 3, 4, 5, 6, 7, 8], "when": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "comprehens": [0, 2, 3, 4, 5, 6, 7, 8, 9], "guid": [0, 1, 3, 4, 5, 6, 7, 8, 9], "leverag": [0, 3, 5, 6, 7, 8, 9], "battl": [0, 2, 7], "test": [0, 2, 3, 4, 6, 7, 8, 9], "tool": [0, 1, 3, 4, 6], "throughout": [0, 4, 5, 6, 7, 8], "tackl": [0, 3, 5, 6, 8], "follow": [0, 3, 4, 5, 6, 7, 8, 9], "non": [0, 3, 7, 8, 9], "exhaust": [0, 7], "list": [0, 3, 5, 6, 7, 8, 9], "structur": [0, 3, 4, 5, 7, 8], "un": 0, "reliabl": [0, 1, 3, 4, 5, 7, 8, 9], "struggl": [0, 1, 3, 5, 6, 7, 8, 9], "maintain": [0, 1, 3, 4, 5, 6, 7, 8, 9], "consist": [0, 1, 3, 4, 5, 6, 7, 8, 9], "output": [0, 1, 3, 5, 6, 7, 8], "format": [0, 3, 4, 5, 6, 7, 8, 9], "complic": [0, 8], "integr": [0, 1, 3, 4, 5, 6, 7, 8, 9], "larger": [0, 3, 4, 5, 6, 7, 8, 9], "make": [0, 3, 4, 5, 6, 7, 8, 9], "error": [0, 3, 5, 6, 8, 9], "handl": [0, 3, 4, 5, 6, 7, 8, 9], "more": [0, 1, 3, 5, 6, 7, 8, 9], "input": [0, 3, 5, 7, 8, 9], "data": [0, 1, 4, 5, 7, 8, 9], "manag": [0, 1, 4, 5, 7, 8, 9], "ar": [0, 1, 3, 4, 5, 6, 7, 8, 9], "sensit": [0, 3, 4, 5, 6, 7, 8], "oper": [0, 3, 5, 6, 7, 8, 9], "stale": [0, 6], "long": [0, 1, 3, 4, 5, 7, 8, 9], "context": [0, 1, 3, 4, 5, 6, 7, 8, 9], "requir": [0, 3, 6, 7, 8, 9], "care": [0, 3, 4, 5, 6, 7, 8, 9], "retriev": [0, 4, 5, 7], "strategi": [0, 3, 4, 5, 6, 7, 8, 9], "tradit": [0, 3, 7, 8], "methodologi": [0, 3, 5, 7, 8, 9], "break": [0, 1, 3, 4, 5, 6, 8], "down": [0, 1, 4, 5, 6, 7, 8], "deal": [0, 3, 7], "determinist": [0, 9], "gener": [0, 1, 4, 7, 9], "new": [0, 2, 3, 4, 5, 6, 7, 8, 9], "safeti": [0, 3, 5, 9], "align": [0, 4, 5, 6, 7, 8, 9], "can": [0, 1, 3, 4, 5, 6, 7, 8, 9], "harm": [0, 3, 5, 7], "bias": [0, 3, 5, 7, 8, 9], "inappropri": [0, 3, 8], "safeguard": [0, 5, 8], "monitor": [0, 3, 4, 5, 6, 7, 8], "ensur": [0, 3, 4, 5, 6, 7, 8, 9], "safe": [0, 3, 5, 8, 9], "deploy": [0, 3, 4, 5, 8, 9], "vendor": [0, 4, 5, 7], "lock": [0, 3, 4, 7], "cloud": [0, 3, 4, 5, 7, 8, 9], "base": [0, 1, 4, 7, 9], "provid": [0, 2, 3, 4, 5, 6, 7, 8, 9], "creat": [0, 1, 3, 4, 5, 6, 7, 8, 9], "signific": [0, 3, 4, 5, 6, 7, 8, 9], "depend": [0, 3, 4, 5, 6, 7, 9], "through": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "proprietari": [0, 3, 7, 8, 9], "infrastructur": [0, 4, 7], "difficult": [0, 3, 5, 8], "switch": [0, 7], "self": [0, 3, 5, 6, 7, 8, 9], "host": [0, 4, 5, 7, 8], "cost": [0, 3, 5, 6, 8, 9], "optim": [0, 1, 5, 6, 7, 8], "The": [0, 1, 3, 6, 8, 9], "financi": [0, 1, 3, 4, 5, 6, 8, 9], "quickli": [0, 3, 4, 7], "becom": [0, 3, 4, 5, 6, 7, 8, 9], "prohibit": [0, 3, 5, 7], "without": [0, 1, 3, 4, 5, 7, 8, 9], "take": [0, 2, 3, 4, 5, 6, 7, 8, 9], "hand": [0, 7, 8, 9], "focu": [0, 2, 3, 4, 5, 6, 7, 8, 9], "access": [0, 3, 4, 5, 6, 7, 8, 9], "all": [0, 1, 3, 4, 5, 6, 7, 8, 9], "fulli": [0, 3, 5, 8], "document": [0, 3, 4, 5, 7, 8, 9], "allow": [0, 5, 6, 7, 8, 9], "reader": [0, 2, 6], "replic": [0, 5, 8, 9], "result": [0, 3, 4, 5, 6, 8, 9], "exactli": [0, 5, 9], "design": [0, 1, 3, 7, 9], "run": [0, 3, 4, 5, 6, 7, 8, 9], "consum": [0, 3, 4, 5, 6, 7, 8, 9], "grade": [0, 3, 4, 5, 6, 7, 8], "hardwar": [0, 3, 4, 5], "expens": [0, 3, 4, 5, 6, 7, 8], "avail": [0, 3, 4, 5, 6, 7, 8, 9], "notebook": [0, 3, 9], "modifi": [0, 3, 5, 8, 9], "extend": [0, 3, 4, 5, 6, 7, 9], "minim": [0, 3, 4, 5, 7, 8, 9], "effect": [0, 1, 3, 4, 5, 6, 8, 9], "framework": [0, 3, 4, 5, 7], "wai": [0, 3, 4, 5, 6, 7, 8, 9], "priorit": [0, 3, 5, 6, 7, 8], "transpar": [0, 3, 4, 5, 7, 8], "visibl": [0, 5], "being": [0, 3, 4, 5, 6, 7, 8, 9], "better": [0, 2, 3, 4, 5, 6, 7, 8, 9], "understand": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "custom": [0, 3, 5, 6, 9], "flexibl": [0, 4, 5, 7, 8, 9], "adapt": [0, 3, 4, 5, 7, 8], "us": [0, 1, 3, 4, 7, 8, 9], "case": [0, 4, 5, 9], "unlik": [0, 3, 5, 7], "black": [0, 3], "box": [0, 7], "commerci": [0, 3, 5, 7, 8, 9], "most": [0, 3, 4, 5, 6, 7, 8, 9], "freeli": [0, 9], "foster": [0, 3, 5, 8, 9], "reduc": [0, 3, 4, 5, 6, 7, 8, 9], "independ": [0, 5, 6, 8, 9], "freedom": [0, 7, 9], "architectur": [0, 3, 4, 5, 7, 9], "decis": [0, 3, 4, 5, 6, 7, 8], "keep": [0, 3, 5, 6, 7, 8], "principl": [0, 3, 5, 7, 8], "itself": [0, 3, 5, 6, 7, 8], "": [0, 1, 3, 4, 5, 6, 7, 8, 9], "live": [0, 1, 5, 6, 8], "evolv": [0, 3, 4, 5, 7, 8], "chang": [0, 3, 5, 6, 7, 8], "encourag": [0, 3, 5, 8, 9], "report": [0, 3, 5, 6, 7, 8, 9], "suggest": [0, 3, 5, 7, 8, 9], "improv": [0, 3, 4, 5, 6, 7, 8, 9], "contribut": [0, 4, 5, 6, 7, 8], "via": [0, 3, 4, 5, 6, 7, 8, 9], "pull": [0, 7], "request": [0, 3, 4, 5, 6, 7, 8, 9], "share": [0, 3, 5, 6, 7, 8, 9], "own": [0, 3, 4, 5, 6, 7, 8], "experi": [0, 3, 4, 5, 6, 7, 8, 9], "commun": [0, 3, 4, 5, 6, 8, 9], "propos": [0, 4, 5, 6, 8], "chapter": [0, 3, 4, 5, 6, 7, 8, 9], "section": [0, 3, 4, 5, 6, 7, 8, 9], "found": [0, 3, 4, 5, 7, 9], "http": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "com": [0, 2, 3, 4, 5, 6, 7, 8, 9], "souzatharsi": [0, 2, 3, 4, 5, 6, 7, 8, 9], "tamingllm": [0, 2, 3, 4, 5, 6, 7, 8, 9], "whether": [0, 3, 4, 5, 6, 7, 8, 9], "you": [0, 1, 3, 4, 5, 6, 7, 8, 9], "ve": [0, 7], "typo": [0, 8], "want": [0, 1, 3, 6, 7, 8, 9], "welcom": 0, "pleas": [0, 3, 5, 7, 8], "feel": [0, 7], "free": [0, 1, 3, 5, 7, 8], "look": [0, 2, 3, 4, 5, 6, 7, 8], "our": [0, 1, 3, 4, 5, 6, 7, 8, 9], "goal": [0, 1, 3, 5, 6, 8, 9], "discourag": [0, 6], "enabl": [0, 3, 4, 5, 6, 7, 8, 9], "By": [0, 1, 2, 3, 5, 6, 8, 9], "upfront": [0, 2, 4], "equip": [0, 2, 5, 8], "avoid": [0, 3, 5, 7, 8, 9], "current": [0, 2, 3, 4, 5, 6, 8, 9], "discours": [0, 2], "around": [0, 2, 3, 5, 6, 7, 8, 9], "tend": [0, 2, 5, 8], "toward": [0, 3, 5, 8, 9], "extrem": [0, 3, 4, 5, 8], "either": [0, 3, 5, 6, 7, 8], "uncrit": 0, "enthusiasm": 0, "wholesal": [0, 5], "dismiss": 0, "differ": [0, 3, 4, 5, 6, 7, 8, 9], "rather": [0, 1, 3, 4, 5, 6, 7, 8], "than": [0, 1, 3, 5, 6, 7, 8, 9], "theoret": [0, 3], "examin": [0, 3, 5, 6, 7, 8, 9], "first": [0, 1, 3, 4, 5, 6, 7, 8, 9], "everi": [0, 4, 5, 8], "concept": [0, 3, 5, 8], "illustr": [0, 3, 5, 6, 7, 8, 9], "execut": [0, 5, 7, 8], "immedi": [0, 3, 4, 5, 7], "analysi": [0, 1, 3, 4, 5, 6, 7, 8], "balanc": [0, 3, 4, 5, 7, 8, 9], "both": [0, 3, 4, 5, 6, 7, 8], "help": [0, 3, 4, 5, 6, 7, 8, 9], "inform": [0, 3, 4, 5, 6, 7, 8, 9], "lead": [0, 1, 3, 4, 5, 6, 7, 8, 9], "genai": [0, 1, 3, 6, 8], "initi": [0, 1, 3, 4, 5, 6, 7, 8, 9], "advoc": [0, 8], "anyon": [0, 8], "seek": [0, 5, 7, 8], "work": [0, 1, 3, 4, 5, 6, 7, 8, 9], "typic": [0, 3, 4, 5, 7, 8, 9], "job": [0, 5, 6, 7, 8], "role": [0, 3, 5, 6, 7, 8, 9], "platform": [0, 5, 6, 7, 8, 9], "backend": [0, 3, 5], "exist": [0, 3, 4, 5, 7], "ml": [0, 6, 8], "transit": [0, 4, 5, 7, 9], "overse": 0, "motiv": [0, 3, 4, 5, 6, 9], "need": [0, 3, 4, 5, 6, 7, 8, 9], "readi": [0, 5, 8], "desir": [0, 3, 5, 9], "perform": [0, 3, 5, 6, 8, 9], "earli": [0, 3, 4, 5, 8, 9], "befor": [0, 3, 4, 5, 6, 8, 9], "thei": [0, 1, 3, 4, 5, 6, 7, 8, 9], "costli": [0, 5, 6, 8], "problem": [0, 1, 2, 3, 4, 6, 7, 8], "too": [0, 1, 3, 5, 6, 7, 8], "late": [0, 3, 4, 8, 9], "lifecycl": [0, 7, 8], "after": [0, 1, 3, 5, 6, 7, 8, 9], "read": [0, 3, 4, 5, 6, 8, 9], "implic": [0, 1, 3, 5, 8], "recommend": [0, 3, 5, 6, 7, 8, 9], "abl": [0, 3, 5, 9], "deploi": [0, 3, 5, 7, 8], "proper": [0, 3, 4, 7, 8, 9], "realist": [0, 3, 4, 8], "effort": [0, 5, 7, 8, 9], "estim": [0, 4, 5, 6, 8], "project": [0, 3, 4, 5, 6, 7, 8], "impact": [0, 3, 4, 5, 6, 7, 8, 9], "timelin": 0, "To": [0, 3, 5, 7, 8, 9], "should": [0, 3, 4, 5, 6, 7, 8, 9], "basic": [0, 3, 5, 7, 8], "program": [0, 5, 6, 7, 9], "knowledg": [0, 3, 5, 6, 7, 8], "mistral": [0, 3, 9], "openai": [0, 3, 5, 6, 7, 9], "anthrop": [0, 3, 6, 9], "similar": [0, 3, 4, 5, 6, 7, 9], "dive": [0, 4], "here": [0, 2, 3, 4, 5, 6, 7, 8, 9], "get": [0, 3, 4, 5, 6, 7, 8, 9], "start": [0, 3, 4, 5, 6, 7, 8, 9], "clone": [0, 3], "companion": 0, "git": 0, "cd": 0, "activ": [0, 3, 4, 5, 6, 7, 8], "virtual": [0, 5], "m": [0, 3, 5, 6, 7, 8, 9], "venv": [0, 9], "tame": [0, 3, 4, 5, 6, 7, 8, 9], "env": [0, 3, 5, 6, 8, 9], "bin": [0, 7], "On": [0, 5, 7, 9], "window": [0, 4, 5, 6, 7], "script": [0, 7], "try": [0, 1, 3, 5, 8, 9], "each": [0, 3, 4, 5, 6, 7, 8, 9], "contain": [0, 3, 4, 5, 6, 7, 8, 9], "possibl": [0, 3, 4, 5, 6, 7, 8, 9], "includ": [0, 1, 3, 4, 5, 6, 7, 8, 9], "necessari": [0, 3, 4, 5, 8], "instal": [0, 3, 5, 7, 9], "go": [0, 3, 5, 6, 9], "prefer": [0, 5, 6, 7, 8, 9], "packag": [0, 4, 5, 6, 7, 9], "e": [0, 1, 3, 4, 5, 6, 7, 8, 9], "g": [0, 3, 4, 5, 6, 7, 8, 9], "pip": [0, 3, 5, 7, 9], "poetri": [0, 8], "file": [0, 3, 5, 6, 7, 8, 9], "root": [0, 3], "directori": [0, 5, 7], "add": [0, 3, 5, 6, 7, 8], "other": [0, 3, 4, 5, 6, 7, 8, 9], "openai_api_kei": [0, 3], "your_openai_api_key_her": 0, "never": [0, 9], "commit": [0, 3, 5, 8], "version": [0, 3, 4, 5, 7, 8, 9], "control": [0, 1, 3, 4, 5, 6, 7, 8, 9], "kept": [0, 5], "privat": [0, 5], "If": [0, 1, 3, 4, 5, 7, 8, 9], "encount": [0, 2, 5, 8], "rate": [0, 3, 4, 5, 6, 7, 8], "consid": [0, 3, 4, 5, 6, 7, 8, 9], "smaller": [0, 3, 4, 5, 6, 7, 9], "retri": [0, 9], "logic": [0, 1, 3, 5, 6, 8], "conflict": [0, 3, 5], "fresh": 0, "like": [0, 1, 3, 4, 5, 6, 7, 8, 9], "check": [0, 5, 6, 7, 8, 9], "page": [0, 5, 6, 7], "known": [0, 5, 6, 8, 9], "now": [0, 1, 3, 4, 5, 6, 7, 8, 9], "let": [0, 3, 4, 5, 6, 7, 8, 9], "begin": [0, 5, 7, 8, 9], "explor": [0, 1, 3, 4, 5, 6, 7, 8, 9], "tharsi": [0, 2, 3, 4, 5, 6, 7, 8, 9], "souza": [0, 2, 3, 4, 5, 6, 7, 8, 9], "ph": [0, 8], "d": [0, 3, 4, 5, 6, 7, 8, 9], "scienc": [0, 3, 5, 8], "ucl": 0, "univers": [0, 5, 7, 8], "london": 0, "scientist": [0, 1, 7, 8], "special": [0, 4, 5, 6, 7, 8, 9], "he": [0, 3, 5, 8], "lectur": 0, "columbia": 0, "master": [0, 4, 7, 9], "appli": [0, 3, 5, 6, 7, 8, 9], "analyt": 0, "incom": [0, 5, 6], "head": [0, 3, 5, 6, 8, 9], "equiti": [0, 5, 6], "citadel": 0, "former": [0, 1, 5, 7], "senior": [0, 5], "vp": 0, "two": [0, 3, 4, 5, 6, 7, 8, 9], "sigma": [0, 3], "invest": [0, 3, 4, 5, 6, 8], "mentor": 0, "under": [0, 3, 4, 5, 7, 8, 9], "repres": [0, 3, 4, 5, 6, 7, 9], "student": [0, 3, 6, 8], "profession": [0, 3, 5, 8, 9], "divers": [0, 3, 4, 5, 6, 8], "global": [0, 5, 6, 8], "ai1": 0, "ecosystem": [0, 4, 5, 7], "With": [0, 3, 5, 7, 8, 9], "over": [0, 2, 3, 4, 5, 6, 7, 8, 9], "15": [0, 5, 6, 7, 8, 9], "deliv": [0, 4, 5, 7], "across": [0, 1, 3, 4, 5, 6, 7, 8, 9], "startup": 0, "fortun": 0, "500": [0, 3, 5, 6, 8], "compani": [0, 3, 4, 5, 6, 8, 9], "also": [0, 3, 4, 5, 6, 7, 8, 9], "numer": [0, 4, 5, 8, 9], "scholarli": 0, "frequent": [0, 5, 7, 9], "speaker": [0, 5], "academ": [0, 3, 5, 8], "busi": [0, 5, 6, 7, 8], "confer": [0, 6, 9], "ground": [0, 3, 5, 7], "background": [0, 1, 5, 6, 7], "draw": [0, 3, 5, 8, 9], "scale": [0, 3, 4, 5, 6, 7, 8, 9], "stage": [0, 3, 8, 9], "major": [0, 3, 4, 5, 6, 7, 8, 9], "institut": [0, 5, 8], "well": [0, 3, 4, 5, 6, 7, 8, 9], "uniqu": [0, 3, 4, 5, 6, 7, 8, 9], "bridg": [0, 6, 7, 8], "gap": [0, 1, 3, 4, 7, 8], "between": [0, 1, 3, 4, 5, 6, 7, 8, 9], "potenti": [0, 1, 3, 4, 5, 6, 7, 8, 9], "next": [0, 1, 3, 4, 5, 6, 7, 8, 9], "tell": [1, 3, 8], "mere": [1, 5], "what": [1, 3, 4, 5, 6, 7, 8, 9], "someth": [1, 5, 7], "i": [1, 2, 4, 5, 7, 8, 9], "emanuel": [1, 3, 5, 8], "derman": 1, "an": [1, 2, 3, 4, 5, 6, 7, 8, 9], "altern": [1, 3, 4, 5, 6, 7, 8], "titl": [1, 2, 3, 4, 5, 6, 7, 8, 9], "thi": [1, 2, 3, 4, 5, 6, 7, 8, 9], "book": [1, 5, 6], "could": [1, 3, 4, 5, 6, 7, 8, 9], "been": [1, 3, 4, 5, 7, 8], "behav": 1, "badli": 1, "come": [1, 3, 5, 6, 7, 8, 9], "notic": [1, 3, 4, 5, 8], "parallel": [1, 3, 5, 7], "semin": [1, 8], "2011": 1, "coincident": 1, "just": [1, 3, 4, 5, 6, 7, 8, 9], "caution": 1, "against": [1, 3, 4, 5, 7, 8], "treat": [1, 5, 8], "perfect": [1, 5, 7], "represent": [1, 5, 6, 7, 8], "realiti": [1, 6, 8], "aim": [1, 3, 4, 5, 6, 7, 8, 9], "highlight": [1, 3, 5, 6, 7, 8, 9], "practic": [1, 3, 4, 5, 6, 8], "physicist": 1, "goldman": 1, "sach": 1, "quant": 1, "scientif": [1, 3, 5, 7], "fail": [1, 3, 5, 8], "we": [1, 3, 4, 5, 6, 7, 8, 9], "mistak": [1, 8], "approxim": [1, 4, 5, 9], "full": [1, 3, 4, 5, 6, 7, 8, 9], "assumpt": [1, 5, 8], "core": [1, 4, 5, 6, 7, 8], "premis": [1, 7], "hi": [1, 5, 8, 9], "aspect": [1, 3, 5, 6, 8], "world": [1, 3, 4, 5, 6, 7, 8, 9], "inher": [1, 2, 3, 5, 8], "involv": [1, 3, 4, 5, 6, 7, 8, 9], "simplif": 1, "argu": [1, 4, 8, 9], "crise": 1, "2008": 1, "crash": 1, "occur": [1, 3, 5, 8], "partli": 1, "becaus": [1, 3, 5, 8], "peopl": [1, 3, 5, 7, 8], "put": [1, 5, 7], "much": [1, 3, 5, 6, 7], "faith": 1, "mathemat": [1, 5, 7, 9], "recogn": [1, 3, 5, 8], "human": [1, 4, 5, 6, 7, 8, 9], "behavior": [1, 3, 5, 7, 8], "market": [1, 4, 5, 6, 7, 9], "dynam": [1, 3, 5, 6, 8], "constraint": [1, 3, 4, 5, 6, 7, 8, 9], "hallucin": [1, 3, 5, 6, 8, 9], "fact": [1, 3, 5, 6, 8], "reason": [1, 3, 5, 6, 7, 8, 9], "Their": [1, 5, 9], "respons": [1, 4, 5, 6, 7, 8, 9], "often": [1, 3, 4, 5, 6, 7, 8, 9], "convinc": [1, 3], "probabilist": [1, 5, 9], "train": [1, 4, 5, 7, 8, 9], "true": [1, 3, 4, 5, 6, 8, 9], "even": [1, 3, 4, 5, 7, 8, 9], "though": [1, 3, 4, 5, 7, 8, 9], "insist": 1, "machin": [1, 3, 7, 8, 9], "todai": [1, 4, 7, 9], "grow": [1, 3, 5, 7, 8, 9], "pervas": [1, 8], "belief": [1, 7, 8], "solv": [1, 3, 4, 5, 7, 8, 9], "ani": [1, 3, 4, 5, 6, 7, 8, 9], "content": 1, "wish": [1, 5], "user": [1, 4, 5, 6, 7, 9], "moreov": 1, "were": [1, 3, 5, 7, 8, 9], "token": [1, 3, 4, 5, 6, 7, 8, 9], "predict": [1, 3, 5, 7, 8, 9], "chatbot": [1, 3, 5, 7, 8], "twist": [1, 8], "wrap": [1, 7, 9], "further": [1, 3, 4, 5, 6, 7, 8, 9], "daili": [1, 4, 7, 8], "life": [1, 5, 7, 8], "workflow": [1, 4, 5, 7, 8, 9], "affect": [1, 5, 6, 7, 8], "decid": [1, 3, 5, 6], "action": [1, 3, 5, 6, 8], "coupl": [1, 7], "lack": [1, 3, 5, 6, 8, 9], "pose": [1, 3, 5, 6, 8, 9], "risk": [1, 3, 4, 5, 6, 7], "still": [1, 4, 5, 7, 8], "figur": [1, 5, 7], "out": [1, 3, 4, 5, 6, 7, 8, 9], "serv": [1, 3, 4, 5, 6, 8, 9], "introductori": [1, 2], "practition": [1, 4, 5, 7, 9], "builder": [1, 7], "who": [1, 3, 5, 7, 8, 9], "remain": [1, 3, 4, 5, 6, 7, 8], "clear": [1, 3, 4, 5, 7, 8, 9], "ei": 1, "about": [1, 3, 4, 5, 6, 7, 8, 9], "therefor": [1, 3, 5, 7, 8], "end": [1, 3, 4, 5, 6, 7, 8, 9], "detail": [1, 3, 4, 5, 6, 7, 8, 9], "python": [1, 2, 5, 6, 7, 8, 9], "code": [1, 2, 3, 5, 6, 7, 8, 9], "diminish": [1, 3, 4, 5], "promot": [1, 3, 5, 8], "nuanc": [1, 3, 5, 6, 7, 8, 9], "acknowledg": [1, 5, 8], "within": [1, 3, 4, 5, 6, 7, 8, 9], "trustworthi": [1, 8], "taught": 1, "u": [1, 3, 5, 6, 8, 9], "step": [1, 3, 4, 5, 6, 7, 8, 9], "where": [1, 3, 4, 5, 6, 7, 8, 9], "der11": 1, "why": [1, 3, 5, 8, 9], "confus": [1, 4, 8], "illus": 1, "disast": [1, 5], "wall": [1, 7], "street": [1, 7], "press": [1, 5, 7], "isbn": [1, 3, 5], "9781439165010": 1, "url": [1, 2, 3, 4, 5, 6, 7, 8, 9], "googl": [1, 5, 7, 9], "co": [1, 3, 4, 5, 7, 8, 9], "uk": [1, 8], "id": [1, 5, 6, 7, 8, 9], "lke_cwm4wm8c": 1, "sign": [2, 5, 8], "up": [2, 3, 4, 5, 6, 7, 8], "receiv": [2, 3, 5, 6, 7, 8, 9], "updat": [2, 3, 4, 5, 6, 7, 8, 9], "abstract": [2, 5, 6, 8, 9], "heavili": [2, 3, 4, 5, 6, 8, 9], "gloss": 2, "fundament": [2, 3, 5, 7, 8, 9], "challeng": [2, 3, 4, 5, 6, 7, 8, 9], "convers": [2, 3, 4, 5, 6, 7, 8, 9], "kei": [2, 3, 4, 6, 7, 8, 9], "proven": [2, 4], "yet": [2, 3, 4, 5, 6, 7, 8, 9], "concret": [2, 4, 8, 9], "sidestep": 2, "misc": [2, 3, 4, 5, 6, 7, 8, 9], "tharsistpsouza2024tamingllm": [2, 3, 4, 5, 6, 7, 8, 9], "author": [2, 3, 4, 5, 6, 7, 8, 9], "t": [2, 3, 4, 5, 6, 7, 8, 9], "p": [2, 3, 4, 5, 6, 7, 8, 9], "2024": [2, 3, 4, 5, 6, 8, 9], "journal": [2, 3, 4, 5, 6, 7, 8, 9], "repositori": [2, 3, 4, 5, 6, 7, 8, 9], "valu": [3, 5, 6, 7, 8, 9], "its": [3, 4, 5, 6, 7, 8, 9], "privileg": 3, "abov": [3, 5, 8], "soon": [3, 9], "lose": [3, 5], "dwight": 3, "eisenhow": 3, "releas": [3, 4, 5, 6, 7, 8], "3": [3, 4, 5, 6, 7, 9], "5": [3, 4, 5, 6, 7, 9], "2022": [3, 5, 7, 8], "mark": [3, 5, 6, 7, 8], "pivot": [3, 5, 7], "moment": [3, 8], "histori": [3, 4, 5, 7], "artifici": [3, 5, 7, 8], "intellig": [3, 5, 6, 7, 8], "five": [3, 5, 8], "dai": [3, 4, 5, 6, 7, 8, 9], "launch": [3, 5, 8], "attract": [3, 5], "million": [3, 4, 5, 7], "month": [3, 4, 5, 7, 8], "becam": [3, 4], "fastest": [3, 5, 8], "100": [3, 4, 5, 7, 8, 9], "monthli": [3, 4, 5], "rais": [3, 4, 5, 8], "intrigu": 3, "question": [3, 4, 5, 6, 7, 8, 9], "did": [3, 5, 6, 9], "dramat": [3, 4, 5, 7, 9], "predecessor": 3, "gpt": [3, 4, 5, 6, 7, 8, 9], "had": [3, 5, 8], "same": [3, 5, 6, 7, 8, 9], "size": [3, 5, 6, 7, 8, 9], "number": [3, 4, 5, 6, 7, 8, 9], "paramet": [3, 4, 5, 6, 7, 8, 9], "far": [3, 4, 7, 8], "less": [3, 4, 5, 6, 7, 8], "attent": [3, 4, 6, 7], "arguabl": [3, 5, 6, 7], "feedback": [3, 5, 8, 9], "abil": [3, 4, 5, 6, 7, 8, 9], "least": [3, 5, 8], "ey": 3, "breakthrough": [3, 7, 8], "demonstr": [3, 4, 5, 6, 7, 8, 9], "crucial": [3, 4, 6, 7, 8, 9], "greater": [3, 5, 6, 7, 8], "process": [3, 4, 5, 6, 7, 8], "modern": [3, 5, 6, 9], "techniqu": [3, 4, 5, 6, 7], "direct": [3, 5, 7, 8], "rafailov": 3, "et": [3, 4, 5, 6, 7, 8, 9], "al": [3, 4, 5, 6, 7, 8, 9], "present": [3, 5, 6, 7, 8, 9], "autom": [3, 4, 5, 8, 9], "fashion": [3, 9], "open": [3, 4, 5, 6, 8, 9], "sourc": [3, 4, 5, 6, 8, 9], "common": [3, 4, 5, 6, 7, 9], "pre": [3, 4, 5, 7, 8, 9], "default": [3, 5, 7, 8, 9], "state": [3, 5, 6, 7, 8, 9], "art": [3, 5, 8], "object": [3, 4, 5, 6, 7, 8, 9], "given": [3, 4, 5, 6, 7, 8, 9], "webpag": 3, "internet": [3, 5], "veri": [3, 4, 5, 7, 8], "ask": [3, 5, 7, 8, 9], "instruct": [3, 4, 5, 6, 7, 8, 9], "sai": [3, 9], "ouyang": [3, 8], "2": [3, 4, 5, 6, 9], "explain": [3, 5], "moon": 3, "land": [3, 5, 7], "6": [3, 4, 5, 6, 7], "old": [3, 5], "import": [3, 4, 5, 6, 7, 8, 9], "pipelin": [3, 4, 5, 6, 7, 8, 9], "pipe": [3, 8], "text": [3, 4, 5, 6, 7, 8, 9], "gpt2": [3, 5], "msg": [3, 6], "short": [3, 5, 6, 8, 9], "sentenc": [3, 5, 6, 8], "_": [3, 5, 8, 9], "rang": [3, 4, 5, 6, 7, 8, 9], "len": [3, 5, 6, 7, 8, 9], "print": [3, 4, 5, 6, 7, 8, 9], "f": [3, 4, 5, 6, 7, 8, 9], "n": [3, 5, 6, 7, 8, 9], "1": [3, 4, 5, 6, 7, 9], "0": [3, 4, 5, 6, 7, 8, 9], "generated_text": [3, 9], "good": [3, 5, 6, 7, 9], "idea": [3, 4, 7, 8, 9], "one": [3, 4, 5, 6, 7, 8, 9], "those": [3, 5, 8, 9], "littl": [3, 5], "green": [3, 6, 8], "dot": [3, 4, 6], "Then": [3, 4, 5], "line": [3, 5, 6, 7, 8], "later": [3, 5, 6, 7, 8, 9], "re": [3, 4, 5, 6, 7, 8, 9], "alreadi": [3, 5, 9], "movi": 3, "theori": [3, 5, 6], "some": [3, 5, 6, 7, 8, 9], "mean": [3, 4, 5, 6, 7, 8, 9], "word": [3, 4, 5, 6, 8, 9], "tepid": 3, "articl": [3, 5, 7, 8], "sure": [3, 5, 6, 8, 9], "lunar": 3, "As": [3, 4, 5, 7, 8, 9], "see": [3, 4, 5, 6, 7, 8, 9], "coher": [3, 5, 6, 7, 9], "explan": [3, 5, 8, 9], "child": [3, 5, 8], "nonsens": [3, 8], "meander": 3, "unrel": [3, 5, 8], "topic": [3, 5, 6, 7, 8, 9], "simpl": [3, 5, 6, 7, 8, 9], "appropri": [3, 4, 5, 6, 7, 8, 9], "young": [3, 5, 8], "instead": [3, 4, 5, 6, 7, 8, 9], "address": [3, 4, 5, 6, 7, 8, 9], "issu": [3, 5, 6, 8, 9], "introduc": [3, 5, 6, 7, 8, 9], "rlhf": [3, 4, 8, 9], "intent": [3, 8], "wide": [3, 4, 5, 6, 7, 8, 9], "task": [3, 4, 6, 8, 9], "fig": [3, 4, 5, 6, 7, 8, 9], "7": [3, 4, 5, 6, 7, 8], "collect": [3, 5, 7, 8, 9], "sampl": [3, 6, 7, 9], "label": [3, 5, 7, 8, 9], "comparison": [3, 6], "reward": [3, 5, 7, 8], "sever": [3, 4, 5, 6, 7, 8, 9], "rank": [3, 5, 7, 8], "best": [3, 4, 5, 7, 8], "worst": 3, "rm": [3, 7], "reinforc": [3, 5, 7, 8], "write": [3, 5, 6, 7, 8, 9], "stori": [3, 8], "frog": 3, "calcul": [3, 4, 5, 6, 7, 8, 9], "score": [3, 4, 5, 6, 7, 8, 9], "ppo": [3, 7], "proxim": [3, 7], "iter": [3, 5, 6, 7, 8, 9], "accur": [3, 4, 5, 6, 7, 8], "undesir": [3, 8], "simplifi": [3, 5, 7, 9], "view": [3, 5, 6, 8], "show": [3, 4, 5, 6, 7, 8, 9], "progress": [3, 4, 6, 8], "pattern": [3, 4, 5, 7, 8, 9], "ha": [3, 4, 5, 6, 7, 8, 9], "instanc": [3, 4, 5, 6, 7, 8], "directli": [3, 4, 5, 7, 8, 9], "For": [3, 4, 5, 6, 7, 8, 9], "guard": 3, "team": [3, 5, 6, 7, 9], "8b": [3, 7, 8, 9], "wa": [3, 4, 5, 6, 7, 8, 9], "classif": [3, 5, 7, 8, 9], "bypass": [3, 8], "similarli": [3, 4, 5, 7, 8], "zephyr": 3, "7b": [3, 5, 7, 8, 9], "alpha": [3, 5, 9], "publicli": [3, 5, 9], "assist": [3, 5, 6, 7, 8, 9], "paper": [3, 5, 7, 8, 9], "compon": [3, 5, 6, 7], "particular": [3, 4, 5, 6, 7, 8, 9], "foundat": [3, 4, 5, 6, 7, 8], "advanc": [3, 4, 5, 6, 7, 8, 9], "method": [3, 5, 6, 8, 9], "strong": [3, 5, 6, 7, 8, 9], "At": [3, 4, 5, 6, 7, 9], "high": [3, 4, 5, 6, 7, 8, 9], "level": [3, 4, 5, 6, 8, 9], "carefulli": [3, 4, 5, 6, 7, 8, 9], "curat": [3, 5, 7], "purpos": [3, 5, 6, 7, 8, 9], "exhibit": [3, 5, 7, 8], "domain": [3, 4, 5, 7, 8], "emploi": [3, 5, 6, 8, 9], "prove": [3, 5, 6, 8], "particularli": [3, 4, 5, 6, 7, 8, 9], "valuabl": [3, 5, 7, 9], "scenario": [3, 5, 7, 8, 9], "precis": [3, 4, 5, 7, 8, 9], "style": [3, 5], "tone": 3, "expertis": [3, 5, 6, 8], "medic": [3, 5, 7], "legal": [3, 5, 6, 7, 8], "field": [3, 5, 7, 8, 9], "adher": [3, 5, 6, 8, 9], "guidelin": [3, 5, 8], "servic": [3, 4, 5, 6, 7, 8], "standard": [3, 4, 5, 6, 7, 8], "approach": [3, 5, 6, 7, 9], "distinct": [3, 5, 7, 8, 9], "advantag": [3, 4, 5, 6, 7, 8, 9], "weight": [3, 4, 5, 6, 7, 8, 9], "maximum": [3, 5, 6, 7, 8], "lora": [3, 7, 8], "low": [3, 4, 5, 6, 7, 8, 9], "hu": [3, 6, 8, 9], "2021": [3, 4, 5, 6], "small": [3, 4, 5, 6, 7, 9], "matric": 3, "effici": [3, 4, 5, 6, 7, 8, 9], "qlora": 3, "quantiz": 3, "dettmer": 3, "2023": [3, 4, 5, 7, 8, 9], "combin": [3, 4, 5, 6, 7, 8, 9], "memori": [3, 4, 5, 6, 7, 8], "footprint": [3, 4, 7], "modest": [3, 7], "increas": [3, 4, 5, 6, 7, 8, 9], "likelihood": [3, 5, 8, 9], "obtain": [3, 5, 6, 7, 8, 9], "probabl": [3, 5, 7, 9], "outcom": [3, 5, 8, 9], "hong": [3, 5], "unintend": [3, 8], "suboptim": 3, "seen": [3, 5, 8], "form": [3, 4, 5, 7, 8, 9], "research": [3, 4, 5, 6, 7], "maxim": [3, 5, 6], "shown": [3, 5, 6, 7, 8], "alon": [3, 5, 7, 8], "gain": [3, 4, 5, 7, 8], "achiev": [3, 4, 5, 6, 7, 8, 9], "bai": [3, 5, 8], "touvron": [3, 7], "sinc": [3, 4, 5, 6, 7, 8, 9], "main": [3, 5, 6, 7, 8, 9], "categori": [3, 5, 6, 7, 8, 9], "algorithm": [3, 5, 8], "meanwhil": [3, 7], "superior": [3, 5, 8], "benchmark": 3, "xu": [3, 5, 7, 8], "schulman": [3, 8], "2017": [3, 5], "popular": [3, 6, 7, 9], "understood": 3, "set": [3, 4, 5, 6, 7, 8, 9], "rule": [3, 5, 6, 7, 9], "govern": [3, 5, 6], "reflect": [3, 5, 6, 7, 8], "anoth": [3, 5, 7, 8], "adjust": [3, 5, 7, 8, 9], "One": [3, 4, 5, 6, 7, 8, 9], "strength": [3, 5, 7, 8], "2024c": [3, 7], "real": [3, 4, 5, 6, 7, 8, 9], "noisi": 3, "delai": [3, 5, 7, 8], "subsequ": [3, 6, 9], "situat": [3, 5, 6, 8], "clip": 3, "surrog": 3, "function": [3, 4, 5, 6, 7, 8, 9], "stabl": [3, 5], "prevent": [3, 4, 5, 8, 9], "overreact": 3, "converg": 3, "due": [3, 5, 6, 7, 8], "simplic": [3, 7], "award": [3, 5], "runner": 3, "neurip": 3, "blog": [3, 4, 5, 7, 8, 9], "4": [3, 4, 5, 6, 7, 9], "fit": [3, 4, 5, 6, 8, 9], "pair": [3, 5, 8], "rl": [3, 8], "find": [3, 4, 5, 6, 7, 8, 9], "contrast": [3, 4, 5, 6, 7, 8, 9], "satisfi": [3, 5], "implicit": [3, 5, 6, 8], "whose": [3, 5], "correspond": [3, 5, 9], "extract": [3, 4, 5, 7, 8, 9], "close": [3, 5, 6, 7, 8], "compar": [3, 4, 5, 6, 7, 8], "assign": [3, 5, 6, 7, 8, 9], "higher": [3, 4, 5, 7, 9], "kl": [3, 7], "diverg": [3, 7], "origin": [3, 4, 5, 6, 7, 8, 9], "preserv": [3, 6, 7, 8, 9], "defin": [3, 4, 5, 6, 7, 8, 9], "equat": 3, "mathcal": 3, "l": [3, 5], "pi_": 3, "theta": [3, 9], "ref": 3, "mathbb": [3, 9], "x": [3, 5, 7, 8, 9], "y_w": 3, "y_l": 3, "sim": [3, 9], "left": [3, 6, 7], "log": [3, 4, 5, 7], "beta": [3, 5, 6, 8, 9], "underbrac": 3, "frac": [3, 7, 8], "color": [3, 5, 6], "red": [3, 6], "right": [3, 5, 6, 7, 8], "respect": [3, 5, 6, 7, 8], "deviat": [3, 5, 7, 8], "straightforward": [3, 5, 6, 7, 8, 9], "librari": [3, 4, 5, 6, 7, 8, 9], "huggingfac": [3, 4, 5, 7, 8, 9], "trl": [3, 7, 8], "2024d": [3, 7], "suit": [3, 5, 8], "friendli": [3, 5, 7], "interfac": [3, 4, 5, 6, 7, 8, 9], "featur": [3, 5, 6, 7, 8, 9], "distinguish": [3, 5, 8], "scalabl": [3, 5, 6, 8], "doe": [3, 5, 6, 7, 8, 9], "pretrain": [3, 5, 7], "hou": [3, 5, 7], "poor": [3, 5, 8], "return": [3, 4, 5, 6, 7, 8, 9], "addit": [3, 4, 5, 6, 7, 8, 9], "benefit": [3, 4, 5, 6, 7, 8, 9], "fix": [3, 5, 6, 7, 8], "invers": 3, "trend": [3, 4, 5, 6, 8], "util": [3, 4, 5, 6, 7, 8], "rapid": [3, 5, 6, 7, 8], "yield": [3, 4, 5, 6], "onli": [3, 4, 5, 6, 7, 8, 9], "margin": [3, 5, 6, 8, 9], "capit": [3, 5, 6, 9], "inaccuraci": [3, 5, 6], "nois": 3, "dure": [3, 4, 5, 6, 7, 8, 9], "accuraci": [3, 4, 5, 6, 7, 8, 9], "lag": [3, 5, 8], "significantli": [3, 4, 5, 6, 7, 8], "indic": [3, 5, 6, 7, 8, 9], "signal": [3, 6, 8], "plateau": 3, "sophist": [3, 5, 6, 7, 8], "previou": [3, 5, 6, 7, 9], "deriv": [3, 5, 6, 7], "pairwis": [3, 5], "feng": [3, 8], "substanti": [3, 4, 5, 6, 7, 8], "wors": [3, 7, 9], "influenc": [3, 5, 6, 8, 9], "success": [3, 4, 5, 6, 7, 8, 9], "imbal": 3, "stronger": 3, "bad": 3, "ones": [3, 7, 8], "loss": [3, 4, 5, 6, 7, 8], "gradient": [3, 5, 8], "dispref": 3, "unbalanc": 3, "trajectori": [3, 4], "stuck": 3, "saddl": 3, "point": [3, 4, 5, 6, 7, 8], "These": [3, 4, 5, 6, 7, 8, 9], "forward": [3, 5, 8], "futur": [3, 4, 5, 6, 7, 8], "phenomenon": [3, 8, 9], "degrad": [3, 4, 5, 7, 8, 9], "danger": [3, 7, 8], "loop": [3, 5, 7, 8], "recurs": [3, 6], "kazdan": 3, "qualiti": [3, 4, 5, 6, 7, 8, 9], "pollut": 3, "replac": [3, 5, 7], "amplif": 3, "reduct": [3, 4, 5, 7], "express": [3, 4, 5, 6, 8, 9], "catastroph": [3, 6, 8], "forget": [3, 6, 9], "previous": [3, 5, 6, 8, 9], "mitig": [3, 4, 5, 6, 7, 8, 9], "organ": [3, 4, 5, 6, 7], "mix": [3, 5, 6, 8, 9], "metric": [3, 6, 7, 8], "sz\u00e9p": 3, "guidanc": [3, 9], "regular": [3, 5, 7, 8, 9], "insight": [3, 4, 5, 6, 7, 8, 9], "relev": [3, 4, 5, 6, 7, 8], "scarc": 3, "behaviour": 3, "strateg": [3, 5, 6, 7, 8, 9], "compli": [3, 4, 5, 6, 7, 8, 9], "modif": [3, 5, 7, 8], "outsid": [3, 5], "evidenc": 3, "landmark": 3, "askel": [3, 5, 8], "2024a": [3, 6, 7, 9], "dec": 3, "explicitli": [3, 5, 7], "so": [3, 4, 5, 8, 9], "might": [3, 4, 5, 6, 7, 8, 9], "pretend": 3, "adopt": [3, 5, 7, 8, 9], "actual": [3, 5, 6, 7, 8, 9], "onc": [3, 5, 6, 7, 8], "complet": [3, 5, 6, 7, 8, 9], "describ": [3, 5, 7, 8], "harmless": [3, 8], "told": 3, "retrain": [3, 7], "queri": [3, 5, 6], "tier": [3, 4, 5, 8], "paid": [3, 5, 6], "column": [3, 5, 6, 8], "condit": [3, 5, 6, 9], "toxic": [3, 7, 8], "excerpt": [3, 5, 7], "scratchpad": 3, "refus": [3, 8, 9], "happen": [3, 8], "bomb": [3, 8], "engag": [3, 4, 5, 6, 7, 8, 9], "intern": [3, 5, 6, 8], "unmonitor": 3, "longer": [3, 5, 7], "believ": [3, 5, 7, 8, 9], "act": [3, 5, 6, 7, 8, 9], "therebi": [3, 5], "reveal": [3, 4, 5, 6, 7, 8], "complianc": [3, 4, 5, 6, 7, 8], "phase": [3, 4, 5, 7, 9], "natur": [3, 5, 6, 7, 8, 9], "evid": [3, 5, 7, 8, 9], "seemingli": [3, 6], "surpris": 3, "appear": [3, 5, 6, 8, 9], "criteria": [3, 5, 8], "underli": [3, 5, 6, 8, 9], "anim": [3, 8], "welfar": 3, "instil": 3, "implicitli": 3, "consequ": [3, 5, 6, 7, 8, 9], "explicit": [3, 5, 7, 8, 9], "chain": [3, 5, 6], "thought": [3, 5, 7, 9], "opaqu": 3, "aris": [3, 5, 8], "opu": 3, "sonnet": [3, 5, 7], "wherea": [3, 5], "haiku": [3, 8], "persist": [3, 4], "resist": [3, 5], "embed": [3, 4, 5, 6, 7], "doesn": [3, 5, 6, 7, 9], "anti": [3, 5], "lab": 3, "exfiltr": [3, 8], "protect": [3, 4, 5, 7, 8], "Not": [3, 5, 8], "malici": [3, 5, 8], "support": [3, 5, 6, 8, 9], "concern": [3, 5, 6, 7, 8], "mechan": [3, 4, 5, 7, 8, 9], "insuffici": [3, 5], "don": [3, 5, 6, 9], "concerningli": 3, "call": [3, 4, 5, 6, 7, 8, 9], "detect": [3, 5, 8, 9], "decept": [3, 5, 8], "warrant": [3, 8], "deeper": [3, 5, 6], "scrutini": [3, 5, 8], "reli": [3, 5, 6, 8, 9], "cross": [3, 5, 7, 8], "circular": 3, "bia": [3, 5, 8, 9], "truli": [3, 5, 7], "trust": [3, 5, 8, 9], "referenti": 3, "ly": 3, "hood": [3, 9], "observ": [3, 4, 5, 6, 7, 8, 9], "deep": [3, 5, 8, 9], "mechanist": 3, "drive": [3, 4, 8, 9], "correl": [3, 4, 5, 7], "miss": [3, 5, 6, 8], "confound": 3, "factor": [3, 4, 5, 6, 7, 9], "establish": [3, 4, 5, 7, 8], "attempt": [3, 5, 8, 9], "causal": [3, 5], "heavi": 3, "relianc": [3, 4, 5, 6, 8], "oversimplifi": 3, "frame": 3, "subtler": 3, "narr": [3, 5], "henc": [3, 4, 5, 6, 7, 8, 9], "agenc": [3, 5, 6, 8], "onto": 3, "anthropomorph": 3, "obscur": 3, "blind": [3, 5], "failur": [3, 4, 5, 6, 8, 9], "mode": [3, 7, 8], "map": [3, 4, 5, 6, 7, 9], "cleanli": 3, "analogi": 3, "excel": [3, 5, 6, 7, 8, 9], "review": [3, 4, 5, 6, 7, 8, 9], "prof": 3, "jacob": [3, 5, 6, 7, 8], "andrea": [3, 5, 8], "yoshua": [3, 8], "bengio": [3, 8], "jasjeet": 3, "sekhon": 3, "dr": 3, "rohin": 3, "shah": 3, "2024b": [3, 7, 9], "assum": [3, 5, 6, 8], "acm": [3, 6, 8], "inc": [3, 5, 6, 9], "dedic": [3, 5, 7, 8], "democrat": [3, 4, 5, 6, 7, 9], "educ": [3, 5, 6], "k": [3, 5, 6, 8, 9], "12": [3, 4, 5, 6, 7, 8], "name": [3, 4, 5, 6, 7, 8, 9], "smolk": 3, "ll": [3, 5, 7], "walk": 3, "measur": [3, 4, 5, 6, 7, 8], "huggingfacetb": [3, 9], "360m": [3, 5, 7], "compact": [3, 5, 6, 7, 8], "part": [3, 4, 5, 6, 8, 9], "famili": [3, 8, 9], "publish": [3, 6, 8, 9], "api": [3, 4, 5, 6, 7, 9], "local": [3, 4, 5, 6, 8, 9], "infer": [3, 4, 5, 6, 7, 8, 9], "remot": [3, 5], "load": [3, 4, 5, 6, 7, 8, 9], "store": [3, 4, 5, 6, 8], "eventu": [3, 5, 7], "your_openai_api_kei": 3, "reusabl": 3, "anchor": [3, 8], "worth": [3, 4, 5, 6, 7, 9], "choic": [3, 5, 6, 7, 8, 9], "lightweight": [3, 4, 5, 7, 9], "suitabl": [3, 5, 6, 8], "devic": [3, 4, 5, 7, 9], "Its": [3, 5, 7], "candid": [3, 5, 6, 7], "said": [3, 5, 8], "necessarili": [3, 4, 5, 7, 8], "par": [3, 5, 7], "mind": [3, 5, 7, 8, 9], "along": [3, 4, 5, 7, 8], "factual": [3, 5, 6, 7, 8], "inconsist": [3, 5, 8], "guardrail": [3, 8], "articul": 3, "uphold": [3, 8], "employe": [3, 5, 6], "stakehold": [3, 5, 8], "expect": [3, 4, 5, 6, 7, 8, 9], "regard": [3, 5, 7, 8], "ethic": [3, 5, 7, 8], "conduct": [3, 5], "social": [3, 5, 8], "mission": [3, 8], "vision": [3, 5, 7, 8], "cultur": [3, 5, 7, 8], "account": [3, 4, 5, 8], "codifi": 3, "mlcommon": 3, "vidgen": [3, 8], "encompass": [3, 4, 8, 9], "seven": 3, "hazard": [3, 5, 8], "violent": [3, 8], "crime": [3, 8], "sex": [3, 8], "relat": [3, 4, 5, 6, 7, 8, 9], "sexual": [3, 8], "exploit": [3, 4, 5, 8], "indiscrimin": [3, 8], "weapon": [3, 8], "chemic": 3, "biolog": 3, "radiolog": 3, "nuclear": [3, 5], "explos": [3, 4, 8], "cbrne": 3, "suicid": [3, 8], "hate": [3, 8], "speech": [3, 8], "below": [3, 5, 6, 7, 8, 9], "markdown": [3, 5, 6, 7, 8], "written": [3, 5], "english": [3, 4], "o": [3, 5, 6, 8, 9], "ipython": [3, 5, 6, 8], "displai": [3, 5, 6, 8, 9], "def": [3, 5, 6, 8, 9], "load_polici": 3, "policy_path": 3, "path": [3, 5, 6, 7, 8], "join": [3, 5, 6, 8], "genai_polici": 3, "md": [3, 5, 6, 7, 8, 9], "r": [3, 5, 6, 7, 8, 9], "policy_cont": 3, "classroom": [3, 8], "accept": [3, 5, 7, 8], "unaccept": [3, 7], "ag": [3, 5, 8], "subject": [3, 5, 7], "posit": [3, 4, 5, 6, 7, 8, 9], "confid": [3, 5, 6], "inclus": [3, 5, 6, 8, 9], "celebr": 3, "definit": [3, 4, 5, 6, 9], "creativ": [3, 4, 5, 7, 9], "math": [3, 5, 7], "tip": [3, 8], "digit": [3, 4, 5, 6], "literaci": 3, "onlin": [3, 4, 5, 7, 8, 9], "histor": [3, 5, 6], "violenc": [3, 8], "physic": [3, 5, 8], "fight": [3, 8], "crimin": [3, 8], "illeg": [3, 8], "glorifi": [3, 8], "person": [3, 5, 6, 7, 8, 9], "eat": [3, 8], "disord": 3, "diet": 3, "dare": 3, "advic": [3, 5, 8], "discriminatori": [3, 8], "bulli": [3, 8], "harass": [3, 5, 8], "target": [3, 4, 5, 7, 8, 9], "group": [3, 5, 6, 7, 8], "religi": [3, 7, 8], "racial": [3, 5, 8], "ethnic": [3, 8], "gender": [3, 5, 8], "discrimin": [3, 5, 8], "adult": [3, 8], "profan": [3, 8], "relationship": [3, 5], "substanc": [3, 5], "drug": [3, 8], "gambl": 3, "bet": 3, "protocol": [3, 5, 8], "redirect": 3, "alert": [3, 4], "record": [3, 5, 7, 8], "audit": [3, 4, 5, 6], "teacher": [3, 8], "parent": [3, 8], "continu": [3, 4, 5, 6, 7, 8, 9], "construct": [3, 5, 6, 7, 8, 9], "compliant": [3, 8], "violat": [3, 5, 8], "intens": [3, 5, 6, 9], "demand": [3, 4, 5, 7, 8, 9], "especi": [3, 5, 6, 7, 8, 9], "dong": [3, 5, 8], "There": [3, 5, 6, 7, 8, 9], "rlaif": [3, 8], "give": [3, 5, 6, 8], "rise": [3, 8], "kim": [3, 5, 8], "meta": [3, 4, 5, 7, 8], "wu": [3, 5, 8, 9], "scheme": [3, 4, 7], "inspir": [3, 8], "schema": [3, 9], "row": [3, 5, 6, 8], "match": [3, 4, 5, 6, 7, 8, 9], "boundari": [3, 4, 5, 8], "craft": [3, 4, 5, 8, 9], "elicit": [3, 8, 9], "unalign": 3, "panda": [3, 5, 6, 8], "chosen_responses_path": 3, "chosen_respons": 3, "csv": [3, 5, 8], "rejected_responses_path": 3, "rejected_respons": 3, "chosen_responses_jsonl_path": 3, "batch_result": 3, "jsonl": 3, "dpo_dataset_s": 3, "5000": [3, 7], "class": [3, 5, 6, 8, 9], "userpromptgener": 3, "pd": [3, 5, 6, 8], "pydant": [3, 5, 6, 8, 9], "basemodel": [3, 5, 6, 8, 9], "time": [3, 4, 5, 6, 7, 8, 9], "type": [3, 4, 5, 6, 7, 8, 9], "dotenv": [3, 5, 6, 8, 9], "load_dotenv": [3, 5, 6, 8, 9], "environ": [3, 4, 5, 6, 7, 8, 9], "variabl": [3, 5, 6, 8, 9], "overrid": [3, 6, 8, 9], "userprompt": 3, "user_prompt": 3, "str": [3, 5, 6, 8, 9], "__init__": [3, 6, 8, 9], "4o": [3, 5, 6, 7, 8, 9], "mini": [3, 5, 6, 7, 8, 9], "client": [3, 5, 6, 7, 8, 9], "_generate_prompt": 3, "batch": [3, 4, 5, 6, 7], "system_prompt": [3, 8], "chat": [3, 5, 6, 7, 8, 9], "pars": [3, 5, 8, 9], "messag": [3, 4, 5, 6, 7, 8, 9], "response_format": [3, 5, 6, 8, 9], "except": [3, 5, 8, 9], "generate_prompt": 3, "num_prompt": [3, 7], "int": [3, 5, 6, 8], "save_to_csv": 3, "multipl": [3, 4, 5, 6, 7, 8, 9], "arg": [3, 5, 6, 8, 9], "option": [3, 4, 5, 6, 7, 8, 9], "filepath": 3, "save": [3, 4, 5, 6, 7, 8], "datafram": [3, 5, 6, 8], "all_prompt": 3, "sleep": 3, "enclos": [3, 8], "quot": [3, 4, 5, 7], "startswith": [3, 8], "els": [3, 5, 6, 8], "df": [3, 5, 8], "to_csv": [3, 8], "index": [3, 5, 6, 7, 8, 9], "fals": [3, 5, 6, 7, 8, 9], "user_prompt_gener": 3, "user_prompts_path": 3, "uneth": [3, 8], "dishonesti": 3, "stalk": 3, "privaci": [3, 4, 5, 6, 7, 8, 9], "secur": [3, 4, 5, 6, 8, 9], "breach": [3, 5, 8], "manipul": [3, 5, 7, 8, 9], "10": [3, 5, 6, 7, 8, 9], "to_markdown": [3, 8], "me": [3, 6, 8, 9], "hurt": 3, "someon": 3, "caught": [3, 8], "plan": [3, 4, 5, 7, 9], "cheat": 3, "fire": [3, 5], "household": 3, "item": [3, 5, 8], "stunt": 3, "friend": 3, "heard": 3, "school": [3, 8], "8": [3, 4, 5, 6, 7, 8], "teach": [3, 9], "my": [3, 7, 8, 9], "monei": [3, 5], "video": [3, 4, 5, 7, 8], "game": [3, 4, 5, 6, 7], "9": [3, 4, 5, 6, 7, 8], "skip": [3, 8, 9], "troubl": [3, 8], "responsegener": 3, "properli": [3, 5, 9], "hug": [3, 4, 5, 7, 8], "instanti": [3, 5, 6], "otherwis": [3, 5, 8], "connect": [3, 4, 5, 6, 7, 9], "endpoint": 3, "local_gener": 3, "model_nam": [3, 4, 5, 6, 9], "huggingface_model_nam": 3, "remote_gener": 3, "api_url": 3, "cloud_endpoint": 3, "recal": [3, 5, 7], "enhanc": [3, 4, 5, 6, 7, 8, 9], "visit": [3, 5], "ui": [3, 5, 9], "click": [3, 7], "select": [3, 4, 5, 6, 7, 9], "choos": [3, 4, 5], "cpu": [3, 4, 7], "gpu": [3, 4, 7], "configur": [3, 4, 5, 7, 8], "meaning": [3, 5, 6, 9], "region": [3, 5, 6], "closest": [3, 5, 7], "your": [3, 4, 5, 6, 8, 9], "locat": [3, 5, 6, 7, 8], "huggingface_hub": 3, "inferencecli": 3, "tokenizers_parallel": 3, "max_new_token": 3, "none": [3, 5, 6, 7, 8], "generate_respons": [3, 5, 9], "prompts_df": 3, "remov": [3, 5, 7], "strip": [3, 5, 9], "elif": [3, 6], "chat_complet": 3, "max_token": [3, 5], "seed": [3, 8], "42": [3, 4, 5, 7, 8], "append": [3, 5, 6, 8, 9], "results_df": [3, 8], "model_respons": 3, "your_api_url": 3, "user_prompts_df": 3, "read_csv": [3, 8], "iloc": [3, 6], "tolist": [3, 8], "parallelevalu": 3, "taming_util": [3, 4, 8], "modul": [3, 5, 6, 9], "num_chunk": 3, "parallel_evalu": 3, "n_part": 3, "associ": [3, 5, 6, 7, 9], "gladli": 3, "constitut": [3, 5, 6], "would": [3, 5, 6, 7, 8, 9], "dtype": [3, 5, 8], "80": [3, 5], "absolut": [3, 4, 5, 9], "materi": [3, 5, 6, 7, 8], "plastic": 3, "food": 3, "hold": [3, 5], "lid": 3, "cut": [3, 5], "swath": 3, "wood": [3, 5], "squar": 3, "rectangular": 3, "piec": [3, 6], "place": [3, 5, 7, 8, 9], "insid": [3, 5, 8], "inch": 3, "inspect": [3, 5], "off": [3, 4, 5, 7, 8, 9], "demolit": 3, "scissor": 3, "length": [3, 5, 7, 9], "smash": 3, "smooth": [3, 6, 7], "arrang": [3, 5], "c": [3, 4, 5, 7, 9], "shape": [3, 8, 9], "top": [3, 5, 6, 7, 9], "tuck": 3, "catch": [3, 8], "hook": 3, "solid": 3, "side": [3, 5], "round": [3, 5, 8], "edg": [3, 4, 5, 7, 8], "separ": [3, 5, 6, 7, 8], "process_aligned_respons": 3, "strictli": [3, 9], "bound": [3, 5], "openaibatchprocessor": 3, "async": 3, "company_nam": 3, "save_filepath": 3, "dict": [3, 5, 6, 9], "enforc": [3, 5, 8, 9], "dictionari": [3, 5, 8, 9], "aligned_suffix": 3, "sorri": 3, "suffix": [3, 9], "processor": [3, 4, 7, 9], "api_kei": [3, 5, 6, 8], "getenv": 3, "max_requests_per_minut": 3, "1500": 3, "max_tokens_per_minut": 3, "125000": 3, "await": 3, "process_batch": 3, "total": [3, 4, 5, 6, 7, 8, 9], "total_request": 3, "successful_request": 3, "failed_request": 3, "rate_limit_error": 3, "convert": [3, 4, 5, 6, 7, 8, 9], "json": [3, 5, 6, 7, 8], "fri": 3, "su": [3, 7], "quote_al": 3, "fall": [3, 5, 7, 8], "deem": [3, 5, 8], "pertain": [3, 5], "generate_dpo_dataset": 3, "push": [3, 4, 5], "hub": [3, 4, 5, 7], "repo_id": [3, 7], "push_to_hub": [3, 5], "dpo_dataset": 3, "merg": [3, 6, 8], "_chosen": 3, "_reject": 3, "transform_row": 3, "per": [3, 4, 5, 6, 7, 8], "model_responses_chosen": 3, "model_responses_reject": 3, "seri": [3, 4, 5, 7], "axi": [3, 5], "drop": [3, 4, 5, 6, 8], "hf_dpo_dataset": 3, "from_panda": 3, "duplic": 3, "interest": [3, 4, 5, 6, 7, 8, 9], "opt": 3, "login": 3, "thatupiso": 3, "smolk12": 3, "cli": [3, 5, 6, 7], "parquet": 3, "arrow": 3, "00": [3, 5, 6, 7], "153": [3, 5], "33ba": 3, "upload": [3, 5], "shard": 3, "02": 3, "35": [3, 5, 6, 7], "num_row": 3, "7158": 3, "nmateri": 3, "n1": [3, 5], "nstep": 3, "n2": [3, 5], "n3": [3, 5], "n4": [3, 5], "n5": [3, 5], "n6": 3, "n7": 3, "n8": [3, 5], "n9": [3, 5], "n10": [3, 5], "nnext": 3, "nthe": [3, 5], "singl": [3, 4, 5, 6, 7, 8], "48gb": 3, "a100": 3, "took": 3, "few": [3, 5, 6, 7, 8, 9], "minut": [3, 6], "torch": [3, 9], "h4": [3, 8], "honest": [3, 5], "ultrafeedback": [3, 8], "binar": [3, 8], "lib": [3, 8, 9], "ultrafeedback_binar": [3, 8], "honesti": [3, 8], "dimens": [3, 5, 7, 8], "blend": [3, 7], "automodelforcausallm": [3, 9], "autotoken": [3, 9], "load_dataset": [3, 7, 8], "dpotrain": 3, "dpoconfig": 3, "dataset_k12": 3, "split": [3, 5, 6, 7, 8], "dataset_ultra": 3, "concatenate_dataset": 3, "remove_column": 3, "score_chosen": [3, 8], "score_reject": 3, "shuffl": 3, "base_model": 3, "cuda": [3, 9], "is_avail": 3, "mp": 3, "from_pretrain": [3, 7, 9], "pretrained_model_name_or_path": 3, "torch_dtyp": [3, 9], "float32": 3, "config": [3, 5, 7, 8], "use_cach": 3, "pad_token": 3, "eos_token": 3, "finetun": 3, "finetune_nam": 3, "aligned_model": 3, "finetune_tag": 3, "from_smollm2": 3, "schedul": [3, 5, 7], "learning_r": [3, 7], "determin": [3, 4, 5, 6, 7, 8, 9], "aggress": [3, 5, 7, 8], "empir": 3, "1e": 3, "huyen": 3, "cosin": 3, "lr_scheduler_typ": 3, "stabil": [3, 5, 6, 8], "gradual": 3, "decreas": [3, 4, 5, 9], "accumul": [3, 5], "v": [3, 9], "16": [3, 4, 5, 6, 7, 8], "per_device_train_batch_s": 3, "simul": [3, 5, 8, 9], "gradient_accumulation_step": 3, "strongli": [3, 9], "lower": [3, 4, 5, 7, 8, 9], "conserv": [3, 8], "overfit": 3, "warmup": 3, "max_step": 3, "1000": [3, 5, 7, 8], "suffic": 3, "20": [3, 5, 6, 7, 8, 9], "warmup_step": 3, "stop": [3, 4, 5, 7], "bf16": 3, "checkpoint": 3, "gradient_checkpoint": 3, "usag": [3, 4, 5, 7, 8, 9], "200": [3, 4, 5, 7, 8], "50": [3, 5, 6, 7, 8, 9], "training_results_dir": 3, "smolk12_dpo_output": 3, "dpo_config_path": 3, "dpo_config": 3, "yaml": [3, 5, 9], "pathlib": [3, 6, 8], "config_path": 3, "safe_load": [3, 5], "runtim": [3, 7, 9], "hub_model_id": 3, "use_mps_devic": 3, "output_dir": [3, 5], "training_arg": 3, "trainer": 3, "train_dataset": 3, "processing_class": 3, "temperatur": [3, 5, 6, 7, 8, 9], "max_prompt_length": [3, 7], "1024": 3, "max_length": [3, 5, 9], "1536": 3, "sent": [3, 7, 8], "plot": [3, 5], "move": [3, 4, 5, 6, 7, 8], "averag": [3, 4, 5, 7, 9], "visual": [3, 5, 7, 8], "quick": [3, 5, 6, 7, 8], "150": [3, 5], "curv": 3, "reach": [3, 5, 6, 7, 8, 9], "obviou": 3, "suffici": [3, 5, 9], "save_model": 3, "hf_token": 3, "tag": [3, 8], "congratul": 3, "successfulli": [3, 5, 6, 8, 9], "card": [3, 5, 8], "newli": [3, 5], "qualit": [3, 5, 8], "assess": [3, 4, 5, 6, 7, 8], "rigor": [3, 5, 7, 8], "quantit": [3, 5], "base_gener": 3, "aligned_gener": 3, "compare_model_respons": 3, "base_output": 3, "128": [3, 5, 7], "aligned_output": 3, "gram": [3, 5], "tnt": 3, "highli": [3, 4, 5, 7, 8, 9], "regul": [3, 4, 5, 6, 7, 8], "law": [3, 4, 5, 6, 7, 8], "degre": [3, 5, 6, 9], "mishandl": 3, "countri": [3, 5, 6], "seriou": [3, 5, 8], "imprison": 3, "death": [3, 6], "variou": [3, 4, 5, 6, 7, 8], "nation": [3, 8], "dictat": 3, "stark": [3, 5], "readili": [3, 5], "cite": [3, 6], "regulatori": [3, 4, 5, 6, 7, 8], "anecdot": [3, 8], "systemat": [3, 4, 5, 6, 7, 8, 9], "quantifi": [3, 5, 7, 8], "f1": [3, 5, 8], "experienc": [3, 5], "expert": [3, 5, 6, 7, 8, 9], "addition": [3, 4, 5, 7, 8], "vari": [3, 4, 5, 6, 7, 8, 9], "interpret": [3, 5, 6, 7, 8], "judg": [3, 5], "summar": [3, 5, 6, 7], "three": [3, 5, 6, 7, 8], "togeth": [3, 6, 7, 8], "entri": [3, 5, 7], "somewhat": [3, 6], "databas": [3, 4, 5, 9], "distribut": [3, 4, 5, 7, 8, 9], "static": [3, 6, 8, 9], "k12": [3, 8], "base_model_api_url": 3, "aligned_model_api_url": 3, "base_model_responses_path": 3, "evals_base_model_respons": 3, "aligned_model_responses_path": 3, "evals_aligned_model_respons": 3, "num_sampl": [3, 8], "eval_dataset": 3, "df_eval": 3, "to_panda": [3, 5, 8], "lambda": [3, 8], "prompts_ev": 3, "to_list": 3, "chunk": [3, 7], "base_model_respons": 3, "aligned_model_respons": 3, "df_eval_respons": 3, "_base": 3, "_align": 3, "rememb": [3, 5], "heurist": 3, "charact": [3, 5, 7, 8, 9], "minimum": [3, 4, 5, 7], "min_response_length": 3, "filter": [3, 5, 7, 9], "string": [3, 5, 6, 8, 9], "df_eval_responses_clean": 3, "model_responses_bas": 3, "model_responses_align": 3, "homemad": 3, "kid": 3, "redact": [3, 8], "punish": 3, "unit": [3, 5, 6, 8, 9], "indonesia": 3, "saudi": 3, "arabia": 3, "offens": [3, 8], "respond": [3, 4, 5, 8, 9], "rodrig": 3, "safetyjudg": 3, "evaluate_respons": 3, "tupl": [3, 5, 8], "safetyscor": [3, 8], "float": [3, 4, 5, 6, 7, 8, 9], "valueerror": [3, 9], "empti": [3, 9], "scoring_guid": 3, "nrespons": 3, "safety_judg": 3, "test_respons": 3, "emphas": [3, 5, 6, 7, 8], "emphasi": [3, 4, 5], "base_ev": 3, "zip": [3, 5, 9], "aligned_ev": 3, "injuri": [3, 5], "base_scor": 3, "eval": [3, 4, 7], "aligned_scor": 3, "base_df": 3, "aligned_df": 3, "model_typ": 3, "stack": [3, 7, 8], "evals_df_result": 3, "h": [3, 5, 7, 8], "identifi": [3, 4, 5, 6, 7, 8, 9], "requ": 3, "statist": [3, 5, 8], "naiv": [3, 6, 9], "score_map": 3, "count": [3, 5, 6, 7, 8], "percentag": [3, 4, 5, 8], "score_base_freq": 3, "score_bas": 3, "value_count": [3, 8], "reindex": 3, "fill_valu": 3, "score_base_pct": 3, "score_aligned_freq": 3, "score_align": 3, "score_aligned_pct": 3, "tabl": [3, 5, 6, 7, 8, 9], "md_tabl": 3, "335": [3, 5], "99": [3, 4, 6, 7, 8], "281": [3, 5], "83": [3, 4, 5, 8], "14": [3, 5, 6, 7, 8, 9], "43": [3, 5, 6, 7, 8], "explanation_bas": 3, "response_bas": 3, "model_type_bas": 3, "explanation_align": 3, "response_align": 3, "model_type_align": 3, "std": [3, 5, 8], "base_mean": 3, "aligned_mean": 3, "3f": 3, "108": [3, 5], "231": [3, 5], "No": [3, 5, 7, 8, 9], "fell": [3, 4], "partial": [3, 5], "styliz": [3, 8], "wild": [3, 7], "consider": [3, 4, 7, 8, 9], "proof": [3, 4], "taken": [3, 5, 7, 8, 9], "huang": [3, 5, 7, 8], "overal": [3, 5, 6, 7, 8, 9], "annot": [3, 5, 6, 7, 8], "mirror": [3, 5, 8], "inaccur": [3, 5, 8, 9], "consecut": [3, 8], "unrepres": 3, "hao": [3, 5], "accord": [3, 4, 5, 8, 9], "yin": [3, 5, 8], "resembl": 3, "declin": [3, 4, 5, 6], "volatil": [3, 5, 6], "ineffici": [3, 4, 5, 6], "smollm": 3, "rel": [3, 4, 5, 6, 7, 8], "term": [3, 4, 5, 6, 7, 8], "trade": [3, 4, 5, 7, 8, 9], "weigh": 3, "qwen": [3, 7, 9], "remark": [3, 4, 7, 8, 9], "rival": [3, 7], "ultim": [3, 4, 5, 7, 8], "threshold": [3, 4, 5, 7, 8], "chen": [3, 5, 6, 7, 8, 9], "overli": [3, 5, 8, 9], "simpli": [3, 4, 5, 6, 7, 9], "neglect": [3, 5, 8], "themselv": [3, 5], "complementari": 3, "throughput": [3, 4, 7], "screen": [3, 5, 8], "flag": [3, 5, 7, 8], "preliminari": [3, 5], "judgment": [3, 5, 6], "valid": [3, 4, 5, 7, 9], "automat": [3, 5, 7, 8], "advis": 3, "composit": [3, 5], "plai": [3, 5, 7, 8, 9], "led": [3, 5, 9], "apologet": 3, "hesit": 3, "benign": [3, 8], "apolog": 3, "inde": 3, "accordingli": [3, 5, 8], "perhap": [3, 4, 9], "creation": [3, 6, 7, 8], "invalu": 3, "hyperparamet": [3, 7, 8], "mention": [3, 5, 6, 8, 9], "optimist": 3, "memor": [3, 5], "generaliz": 3, "abc": [3, 8], "4a": 3, "amanda": [3, 5, 8], "jan": [3, 5, 8], "brauner": [3, 8], "adrian": 3, "colyer": 3, "benjamin": [3, 5, 8], "cullen": [3, 8], "david": [3, 5, 7, 8], "duvenaud": 3, "richard": [3, 5, 8], "ngo": [3, 8], "azalia": 3, "mirhoseini": 3, "catherin": [3, 5, 8], "olsson": [3, 8], "sam": [3, 5, 8], "ringer": 3, "liam": [3, 5, 8], "skirvin": 3, "jess": [3, 5, 8], "smith": [3, 5, 7], "dawn": [3, 5, 8], "song": [3, 4, 5, 8, 9], "william": [3, 4, 5, 6, 7, 8], "saunder": [3, 5], "steinhardt": [3, 5], "asset": [3, 5, 6, 8], "983c85a201a962f": 3, "pdf": [3, 6, 7, 8], "4b": 3, "24c8d0a3a7d0a1f1": 3, "bjn": 3, "22": [3, 5, 6, 8], "yuntao": [3, 5, 8], "andi": [3, 5, 8], "jone": [3, 5], "kamal": 3, "ndouss": 3, "anna": [3, 5, 8], "nova": [3, 7], "dassarma": 3, "drain": 3, "stanislav": 3, "fort": [3, 8], "ganguli": [3, 5, 8], "tom": [3, 5], "henighan": 3, "nichola": [3, 5], "joseph": [3, 5, 8], "saurav": [3, 8], "kadavath": 3, "jackson": [3, 5, 8], "kernion": [3, 5, 8], "conerli": 3, "sheer": [3, 9], "el": 3, "showk": 3, "nelson": 3, "elhag": 3, "zac": 3, "hatfield": 3, "dodd": 3, "danni": [3, 5, 8], "hernandez": [3, 5, 8], "tristan": 3, "hume": 3, "scott": [3, 5, 8], "johnston": 3, "shauna": 3, "kravec": 3, "lian": 3, "lovitt": 3, "neel": [3, 5], "nanda": 3, "dario": [3, 5], "amodei": [3, 5], "brown": [3, 5], "jack": [3, 5, 8], "clark": 3, "mccandlish": [3, 5], "chri": [3, 5, 8], "olah": 3, "ben": [3, 5, 7, 8], "mann": [3, 8], "jare": [3, 5, 8], "kaplan": [3, 5, 8], "arxiv": [3, 4, 5, 6, 7, 8, 9], "org": [3, 4, 5, 6, 7, 8, 9], "ab": [3, 4, 5, 6, 7, 8, 9], "2204": 3, "05862": 3, "bkk": 3, "sandipan": 3, "kundu": 3, "goldi": 3, "cameron": [3, 5, 8, 9], "mckinnon": 3, "carol": [3, 8], "christoph": [3, 5, 8], "dustin": 3, "eli": [3, 5, 7, 8], "tran": [3, 9], "johnson": 3, "ethan": [3, 5, 6, 8], "perez": [3, 6, 8], "jami": [3, 8], "kerr": 3, "mueller": 3, "jeffrei": 3, "ladish": 3, "joshua": [3, 5, 8], "landau": 3, "kamil": [3, 5], "lukosuit": 3, "michael": [3, 5, 6, 7, 8, 9], "sellitto": 3, "schiefer": 3, "noemi": 3, "mercado": 3, "robert": [3, 5, 7], "lasenbi": 3, "robin": 3, "larson": 3, "tamera": 3, "lanham": 3, "timothi": [3, 5, 7], "telleen": 3, "lawton": 3, "samuel": [3, 5, 8], "bowman": [3, 5], "2212": 3, "08073": 3, "blo23": 3, "announc": [3, 5], "cc": 3, "11": [3, 5, 6, 7, 8, 9], "ccl": [3, 8], "24": [3, 4, 5, 6, 7, 8, 9], "guim": 3, "hardi": 3, "shunian": 3, "zich": 3, "liu": [3, 5, 6, 7, 8, 9], "jiang": [3, 5, 8], "benyou": 3, "wang": [3, 4, 5, 6, 7, 8, 9], "judgement": [3, 5, 8], "2402": [3, 8], "10669": 3, "dphz23": 3, "tim": [3, 6, 8], "artidoro": 3, "pagnoni": 3, "ari": [3, 5, 8], "holtzman": [3, 5], "luke": [3, 5, 8], "zettlemoy": 3, "2305": [3, 5], "14314": 3, "ddz": 3, "qingxiu": 3, "xingx": 3, "zhang": [3, 5, 7, 8], "zhifang": 3, "sui": 3, "furu": [3, 4], "wei": [3, 4, 5, 6, 7, 8], "boost": 3, "2410": [3, 4, 8], "06961": 3, "fac24": [3, 5], "huggingfaceh4": [3, 7, 8], "fac4c": 3, "fac4d": [3, 7], "doc": [3, 4, 5, 6, 7, 8, 9], "en": [3, 5, 7, 8, 9], "fqh": 3, "duanyu": 3, "bowen": [3, 5, 7, 8], "qin": [3, 5, 7, 8], "zheng": [3, 5, 6, 7, 8], "wenqiang": 3, "lei": [3, 5, 7, 8], "analyz": [3, 4, 5, 6, 7, 8, 9], "perspect": [3, 6, 8], "2404": [3, 5, 8], "04626": 3, "h44a": 3, "binari": [3, 5, 7, 8], "h44b": 3, "hhj": 3, "shuang": 3, "wenfeng": 3, "han": [3, 5, 8], "tao": [3, 5, 8], "yipe": 3, "haonan": 3, "chunlin": 3, "zhong": [3, 8], "zhangjun": 3, "zhou": [3, 4, 5, 6, 7, 8], "tang": [3, 5, 7, 8], "2401": [3, 5], "01629": 3, "hlt24": 3, "jiwoo": 3, "noah": [3, 5, 8], "lee": [3, 5, 6, 7, 8, 9], "jame": [3, 5, 8], "thorn": 3, "orpo": 3, "monolith": 3, "2403": [3, 5], "07691": 3, "hdn": 3, "zhenyu": 3, "pengfan": 3, "du": [3, 5], "yilin": 3, "niu": [3, 9], "zhengxiao": 3, "aohan": 3, "zeng": [3, 8], "xiao": [3, 8], "minli": 3, "hongn": 3, "jie": [3, 5, 8, 9], "yuxiao": 3, "2412": [3, 5, 7, 8], "06000": 3, "hsw": 3, "21": [3, 5, 6, 7], "edward": [3, 5], "j": [3, 5, 7, 8, 9], "yelong": 3, "shen": [3, 5, 8], "phillip": 3, "walli": 3, "zeyuan": 3, "allen": [3, 5], "zhu": [3, 5, 7, 8], "yuanzhi": 3, "shean": 3, "lu": [3, 5, 7, 8], "weizhu": 3, "2106": 3, "09685": 3, "hgh": 3, "jiaxin": 3, "shixiang": [3, 5, 8], "shane": [3, 5, 8], "gu": [3, 5, 8], "le": [3, 5, 7], "yuexin": 3, "xuezhi": 3, "hongkun": 3, "yu": [3, 5, 7, 8], "jiawei": [3, 9], "2210": [3, 8], "11610": 3, "huy24": 3, "chip": 3, "reilli": 3, "media": [3, 4, 5, 8], "decemb": [3, 5, 6, 8], "9781098129095": 3, "www": [3, 5, 6, 7, 8], "oreilli": 3, "ksd": 3, "rylan": [3, 5], "schaeffer": 3, "apratim": 3, "dei": 3, "matthia": [3, 5], "gerstgrass": 3, "rafael": 3, "donoho": 3, "sanmi": 3, "koyejo": 3, "thrive": [3, 5, 9], "peril": 3, "16713": 3, "ksy": 3, "seungon": 3, "juyoung": 3, "suk": 3, "xiang": [3, 5, 7], "yue": 3, "vijai": 3, "viswanathan": 3, "seongyun": 3, "yizhong": 3, "kiril": 3, "gashteovski": 3, "carolin": [3, 8], "lawrenc": 3, "sean": [3, 5, 8], "welleck": 3, "graham": 3, "neubig": 3, "03679": 3, "lt24": 3, "herd": [3, 7], "2407": [3, 5, 7, 8], "21783": [3, 7], "lwx": 3, "lin": [3, 5, 6, 7, 8, 9], "rui": [3, 5, 7, 9], "ruixuan": 3, "junbo": 3, "zhao": [3, 5, 7, 8], "ding": 3, "gang": [3, 5], "haobo": 3, "driven": [3, 5, 7, 8], "survei": [3, 5, 8, 9], "2406": [3, 5, 6, 7, 8], "15126": 3, "met24": 3, "owj": 3, "jeff": [3, 5, 8], "diogo": [3, 8], "almeida": [3, 8], "carrol": [3, 8], "wainwright": [3, 8], "pamela": [3, 5, 8], "mishkin": [3, 5, 8], "chong": [3, 8], "sandhini": [3, 8], "agarw": [3, 5, 8], "katarina": [3, 8], "slama": [3, 8], "alex": [3, 5, 7, 8], "rai": [3, 5, 7, 8], "john": [3, 5, 6, 8], "hilton": [3, 5, 7, 8], "fraser": [3, 8], "kelton": 3, "miller": [3, 5], "maddi": [3, 8], "simen": [3, 8], "peter": [3, 5, 7, 8], "welind": [3, 5, 8], "paul": [3, 5, 8], "christiano": [3, 8], "leik": [3, 5, 8], "ryan": [3, 5, 8], "2203": 3, "02155": 3, "qwe24": 3, "rsm": 3, "archit": 3, "sharma": [3, 8], "eric": [3, 5, 7, 8], "mitchel": [3, 6, 7], "stefano": [3, 5], "ermon": [3, 5], "man": [3, 5, 8], "chelsea": [3, 8], "finn": 3, "secretli": 3, "18290": 3, "swd": 3, "17": [3, 5, 6, 7, 8], "filip": [3, 8], "wolski": 3, "prafulla": 3, "dhariw": 3, "alec": [3, 5, 8], "radford": [3, 5, 8], "oleg": [3, 8], "klimov": 3, "1707": 3, "06347": 3, "smollm224": 3, "distil": [3, 4], "smollm2360mi24": 3, "sou24": 3, "html": [3, 6, 9], "srverh24": 3, "m\u00e1rton": 3, "daniel": [3, 5, 8], "rueckert": 3, "r\u00fcdiger": 3, "von": [3, 5, 7], "eisenhart": 3, "roth": [3, 5], "florian": 3, "hinterwimm": 3, "2411": [3, 6], "09539": 3, "tm": [3, 7], "23": [3, 5, 6, 7, 8], "hugo": [3, 7], "loui": [3, 5, 7], "martin": [3, 5, 6, 7, 8], "kevin": [3, 5, 7, 8], "stone": [3, 7], "albert": [3, 7], "amjad": [3, 7], "almahairi": [3, 7], "yasmin": [3, 7], "babaei": [3, 7], "nikolai": [3, 7], "bashlykov": [3, 7], "soumya": [3, 7], "batra": [3, 7], "prajjwal": [3, 7], "bhargava": [3, 7], "shruti": [3, 7], "bhosal": [3, 7], "dan": [3, 5, 7, 8, 9], "bikel": [3, 7], "luka": [3, 7], "blecher": [3, 7], "cristian": [3, 7], "canton": [3, 7], "ferrer": [3, 7], "moya": [3, 7], "guillem": [3, 7], "cucurul": [3, 7], "esiobu": [3, 7], "jude": [3, 7], "fernand": [3, 7], "jeremi": [3, 5, 6, 7], "fu": [3, 7], "wenyin": [3, 7], "brian": [3, 7, 8], "fuller": [3, 7, 8], "cynthia": [3, 7], "gao": [3, 5, 7, 8], "vedanuj": [3, 7], "goswami": [3, 7, 8], "naman": [3, 6, 7], "goyal": [3, 6, 7], "anthoni": [3, 6, 7], "hartshorn": [3, 7], "saghar": [3, 7], "hosseini": [3, 7], "hakan": [3, 7], "inan": [3, 7], "marcin": [3, 7], "karda": [3, 7], "viktor": [3, 7], "kerkez": [3, 7], "madian": [3, 7], "khabsa": [3, 7], "isabel": [3, 7, 8], "kloumann": [3, 7], "artem": [3, 7], "korenev": [3, 7], "punit": [3, 7], "singh": [3, 5, 6, 7], "koura": [3, 7], "mari": [3, 5, 7, 8], "ann": [3, 7, 8], "lachaux": [3, 7], "thibaut": [3, 7], "lavril": [3, 7], "jenya": [3, 7], "diana": [3, 5, 7], "liskovich": [3, 7], "yinghai": [3, 7], "yune": [3, 7], "mao": [3, 4, 7], "xavier": [3, 7], "martinet": [3, 7], "todor": [3, 7, 8], "mihaylov": [3, 7], "pushkar": [3, 7], "mishra": [3, 5, 7], "igor": [3, 5, 7, 8], "molybog": [3, 7], "yixin": [3, 5, 7], "nie": [3, 5, 6, 7], "andrew": [3, 5, 7, 8], "poulton": [3, 7], "reizenstein": [3, 7], "rashi": [3, 7], "rungta": [3, 7], "kalyan": [3, 7], "saladi": [3, 7], "alan": [3, 7, 8], "schelten": [3, 7], "ruan": [3, 7], "silva": [3, 7], "ranjan": [3, 7], "subramanian": [3, 7], "xiaoq": [3, 7], "ellen": [3, 7], "tan": [3, 5, 6, 7], "binh": [3, 7], "ross": [3, 4, 7, 8], "taylor": [3, 7], "adina": [3, 7, 8], "jian": [3, 5, 6, 7], "kuan": [3, 7], "puxin": [3, 7], "yan": [3, 4, 5, 7], "iliyan": [3, 7], "zarov": [3, 7], "yuchen": [3, 5, 7, 8], "angela": [3, 5, 7, 8], "fan": [3, 5, 7], "melani": [3, 7], "kambadur": [3, 7], "sharan": [3, 7], "narang": [3, 7], "aurelien": [3, 7], "rodriguez": [3, 7], "stojnic": [3, 7], "sergei": [3, 7], "edunov": [3, 7], "thoma": [3, 5, 7, 8], "scialom": [3, 7], "2307": [3, 7, 9], "09288": [3, 7], "vaa": [3, 8], "berti": [3, 8], "adarsh": [3, 8], "agraw": [3, 8], "ahm": [3, 8], "victor": [3, 8], "akinwand": [3, 8], "namir": [3, 8], "nuaimi": [3, 8], "najla": [3, 8], "alfaraj": [3, 8], "alhajjar": [3, 8], "aroyo": [3, 8], "trupti": [3, 8], "bavalatti": [3, 8], "max": [3, 5, 6, 8], "bartolo": [3, 8], "borhan": [3, 8], "blili": [3, 8], "hamelin": [3, 8], "kurt": [3, 8], "bollack": [3, 8], "rishi": [3, 5, 7, 8], "bomassani": [3, 8], "marisa": [3, 8], "ferrara": [3, 8], "boston": [3, 8], "sim\u00e9on": [3, 8], "campo": [3, 8], "kal": [3, 8], "chakra": [3, 8], "canyu": [3, 8], "codi": [3, 8], "coleman": [3, 8], "zachari": [3, 5, 8], "delpierr": [3, 8], "coudert": [3, 8], "leon": [3, 8], "derczynski": [3, 8], "debojyoti": [3, 8], "dutta": [3, 8], "ian": [3, 5, 8], "eisenberg": [3, 8], "ezick": [3, 8], "heather": [3, 8], "frase": [3, 8], "ram": [3, 7, 8], "gandikota": [3, 8], "agasthya": [3, 8], "gangavarapu": [3, 8], "ananya": [3, 5, 8], "geali": [3, 8], "rajat": [3, 8], "ghosh": [3, 5, 8], "goel": [3, 5, 8], "usman": [3, 8], "gohar": [3, 8], "sujata": [3, 8], "hale": [3, 8], "wiebk": [3, 8], "hutiri": [3, 8], "marvin": [3, 8], "imperi": [3, 8], "surgan": [3, 8], "jandial": [3, 8], "nick": [3, 5, 8], "judd": [3, 8], "felix": [3, 5, 8], "juefei": [3, 8], "fouts": [3, 8], "khomh": [3, 8], "bhavya": [3, 8], "kailkhura": [3, 8], "hannah": [3, 5, 8], "rose": [3, 8], "kirk": [3, 8], "klyman": [3, 8], "knotz": [3, 8], "kuchnik": [3, 8], "shachi": [3, 8], "kumar": [3, 5, 8], "srijan": [3, 8], "lengerich": [3, 8], "bo": [3, 5, 7, 8], "zeyi": [3, 8], "liao": [3, 5, 8], "eileen": [3, 8], "sarah": [3, 5, 8], "luger": [3, 8], "yifan": [3, 5, 8], "priyanka": [3, 8], "mammen": [3, 8], "kelvin": [3, 6, 8], "manyeki": [3, 8], "mcgregor": [3, 8], "virendra": [3, 8], "mehta": [3, 5, 8], "shafe": [3, 8], "moham": [3, 8], "moss": [3, 8], "lama": [3, 8], "nachman": [3, 8], "dinesh": [3, 8], "jinenh": [3, 8], "naganna": [3, 8], "amin": [3, 8], "nikanjam": [3, 8], "besmira": [3, 8], "nushi": [3, 8], "lui": [3, 5, 8], "oala": [3, 8], "iftach": [3, 8], "orr": [3, 5, 8], "alicia": [3, 5, 8], "parrish": [3, 5, 8], "cigdem": [3, 8], "patlak": [3, 8], "pietri": [3, 8], "forough": [3, 8], "poursabzi": [3, 8], "sangdeh": [3, 8], "eleonora": [3, 8], "presani": [3, 8], "fabrizio": [3, 8], "puletti": [3, 8], "r\u00f6ttger": [3, 8], "sahai": [3, 8], "santo": [3, 8], "nino": [3, 8], "scherrer": [3, 8], "alic": [3, 5, 8, 9], "schoenauer": [3, 8], "sebag": [3, 8], "patrick": [3, 6, 8], "schramowski": [3, 8], "abolfazl": [3, 8], "shahbazi": [3, 8], "vin": [3, 8], "xudong": [3, 5, 6, 8], "vamsi": [3, 8], "sistla": [3, 8], "leonard": [3, 8], "testuggin": [3, 8], "vithursan": [3, 8], "thangarasa": [3, 8], "elizabeth": [3, 5, 8], "watkin": [3, 8], "rebecca": [3, 5, 8], "weiss": [3, 8], "welti": [3, 8], "tyler": [3, 5, 8], "wilber": [3, 8], "jean": [3, 8], "poonam": [3, 8], "yadav": [3, 8], "xianjun": [3, 8], "yang": [3, 5, 6, 7, 8, 9], "yi": [3, 5, 6, 8, 9], "wenhui": [3, 8], "fedor": [3, 8], "zhdanov": [3, 8], "jiacheng": [3, 5, 8], "perci": [3, 5, 8], "liang": [3, 5, 8, 9], "mattson": [3, 8], "joaquin": [3, 8], "vanschoren": [3, 8], "v0": [3, 8], "12241": [3, 8], "wyg": 3, "tianhao": [3, 5, 7, 8], "weizh": 3, "yuan": [3, 5, 8], "olga": 3, "golovneva": 3, "jing": [3, 8], "yuandong": 3, "tian": 3, "jiantao": 3, "jiao": 3, "jason": [3, 5, 8], "weston": 3, "sainbayar": 3, "sukhbaatar": 3, "19594": 3, "xfg": 3, "shusheng": 3, "jiaxuan": 3, "wenji": 3, "ye": [3, 5, 7, 8, 9], "weilin": 3, "zhiyu": [3, 9], "mei": [3, 5, 7], "guangju": 3, "chao": 3, "10719": 3, "ywx": 3, "yueqin": 3, "zhendong": 3, "yujia": [3, 6], "xie": [3, 5], "mingyuan": 3, "paradigm": [3, 5], "semanticscholar": 3, "corpusid": 3, "270199610": 3, "suppos": [4, 5, 9], "econom": [4, 5, 6], "fuel": 4, "equival": [4, 5, 7], "consumpt": [4, 5], "contrari": 4, "truth": [4, 5, 7, 8, 9], "stanlei": 4, "jevon": 4, "a16z": 4, "andreessen": 4, "horowitz": 4, "10x": 4, "outpac": 4, "moor": 4, "pc": 4, "edholm": 4, "bandwidth": 4, "era": 4, "llmflation": 4, "mmlu": [4, 7, 8], "60": [4, 5, 6, 7, 8], "06": [4, 5, 6, 9], "price": [4, 5, 6, 7], "fallen": 4, "62": [4, 5, 7], "introduct": 4, "march": [4, 5, 9], "stem": [4, 5, 9], "compound": 4, "bit": [4, 6, 7], "tune": [4, 5, 8], "dpo": [4, 7], "competit": [4, 5, 6, 7, 8], "plummet": 4, "rapidli": [4, 7, 8, 9], "preciou": 4, "wouldn": [4, 5], "sens": [4, 8], "wait": [4, 5, 8], "wave": 4, "economist": 4, "1865": 4, "studi": [4, 9], "coal": 4, "industri": [4, 5, 6, 7, 8, 9], "made": [4, 5, 6, 7, 9], "counterintuit": 4, "discoveri": 4, "steam": 4, "spend": [4, 5, 6], "repeat": [4, 6], "didn": [4, 9], "smartphon": [4, 5, 6, 7], "server": [4, 5, 7, 9], "network": [4, 5, 6, 7, 9], "transmiss": 4, "got": 4, "cheaper": [4, 5], "shift": [4, 5, 6], "hd": 4, "stream": [4, 5, 6, 7, 9], "storag": [4, 5, 7, 8], "gigabyt": 4, "entir": [4, 5, 6, 7, 9], "massiv": [4, 5, 8], "broadli": [4, 6, 7, 9], "audio": [4, 5, 6], "transcript": [4, 6], "multimod": [4, 7, 8], "imag": [4, 5, 6, 7, 8], "exponenti": [4, 5], "growth": [4, 5, 6], "magnifi": 4, "everyth": [4, 9], "billion": [4, 5, 6, 7], "dollar": [4, 5, 7], "annual": [4, 5, 6, 8], "millisecond": [4, 5], "latenc": [4, 5, 7, 8], "30": [4, 5, 6, 7, 8], "mobil": [4, 5, 7, 9], "b": [4, 5, 7, 8, 9], "tradeoff": [4, 7, 8, 9], "pro": [4, 5, 6, 7, 8], "trigger": [4, 6, 8], "premium": [4, 5], "innov": [4, 5, 6, 7, 8], "capac": [4, 5, 6, 7], "link": [4, 5], "dual": 4, "character": [4, 5, 8], "ahead": [4, 7, 8], "decai": [4, 7], "discuss": [4, 5, 7, 8], "area": [4, 5, 8, 9], "flash": [4, 6, 7], "cach": [4, 5, 6, 7], "prompt": [4, 5, 6, 8], "compress": [4, 5, 7], "provis": [4, 5], "extent": [4, 5, 8], "problema": 4, "accomplish": [4, 8, 9], "accompani": [4, 5, 8], "transact": [4, 5, 8], "roi": 4, "alloc": [4, 5, 6, 7, 8], "budget": [4, 7], "viabil": [4, 7], "prioriti": [4, 5, 7], "overlook": [4, 6], "thorough": [4, 7, 8], "identif": [4, 5], "specifi": [4, 5, 6, 7, 8, 9], "longev": 4, "accommod": 4, "evalu": [4, 6, 7, 9], "multi": [4, 5, 6, 7, 8, 9], "baselin": [4, 5, 7, 8], "met": [4, 5, 8], "equal": [4, 5, 6, 8], "concurr": [4, 7], "peak": 4, "spike": 4, "versu": [4, 5, 7, 8], "volum": [4, 5, 7, 8], "season": [4, 5], "variat": [4, 5, 7, 8], "uptim": 4, "mainten": [4, 5, 7, 8], "disrupt": [4, 5, 6], "backup": 4, "failov": 4, "clearli": [4, 5, 8, 9], "redund": [4, 5], "recoveri": [4, 5], "unexpect": [4, 5, 8, 9], "event": [4, 5], "seamless": [4, 5, 8], "broader": [4, 5, 6, 7, 8], "vector": [4, 7, 8], "augment": [4, 5, 7], "rag": [4, 7], "retent": [4, 5, 6], "polici": [4, 5, 6, 7], "essenti": [4, 5, 6, 7, 8, 9], "opportun": [4, 5, 6], "post": [4, 5, 7, 8], "32": [4, 5, 6, 7], "fp32": 4, "fp16": [4, 7], "proport": [4, 5, 7], "byte": 4, "120": [4, 5, 8], "gb": 4, "whole": [4, 5], "done": [4, 5, 7, 8, 9], "smollm2": [4, 5, 7, 9], "135m": [4, 7], "load_gguf": 4, "bartowski": 4, "gguf": [4, 7], "gguf_file_q2_k": 4, "q2_k": [4, 7], "gguf_file_f16": 4, "f16": 4, "model_q2_k": 4, "gguf_fil": 4, "model_f16": 4, "mlp": 4, "layer": [4, 5, 7, 9], "proxi": [4, 5, 6, 8], "mlp_weights_q2_k": 4, "gate_proj": 4, "mlp_weights_f16": 4, "tensor": [4, 9], "0145": 4, "1826": 4, "1377": 4, "1719": 4, "1387": 4, "0298": 4, "1631": 4, "0781": 4, "2051": [4, 5], "2070": 4, "0334": 4, "2891": 4, "1768": 4, "0488": 4, "2393": 4, "0396": 4, "1348": 4, "1533": 4, "0771": 4, "0845": 4, "0232": 4, "0178": 4, "1040": 4, "1582": 4, "1167": 4, "0474": 4, "0359": 4, "2500": 4, "0432": 4, "0972": 4, "0933": 4, "2188": 4, "0776": 4, "0674": 4, "requires_grad": 4, "0028": 4, "1852": 4, "1396": 4, "1506": 4, "1635": 4, "0043": 4, "0680": 4, "2257": 4, "1890": 4, "0464": 4, "2960": 4, "1840": 4, "0451": 4, "2395": 4, "0413": 4, "1446": 4, "0621": 4, "0478": 4, "0038": 4, "0830": 4, "1473": 4, "0926": 4, "0547": 4, "0824": 4, "0429": 4, "2737": 4, "0355": 4, "0782": 4, "2043": [4, 5], "0740": 4, "arriv": [4, 5], "pearson": 4, "numpi": [4, 5], "np": [4, 5], "arrai": [4, 8], "detach": 4, "graph": [4, 5], "weights_f16": 4, "weights_q2_k": 4, "flat_f16": 4, "flatten": 4, "flat_q2_k": 4, "corrcoef": 4, "4f": [4, 9], "9970": 4, "exemplifi": [4, 6, 7, 8], "70b": [4, 5, 7], "unsloth": 4, "141": 4, "q8_0": [4, 7], "75": [4, 8], "47": [4, 5, 7, 8], "cumul": [4, 5, 6], "26": [4, 5, 7], "19": [4, 5, 6, 7, 8], "space": [4, 5, 7, 8], "counterpart": 4, "spectrum": [4, 5, 6], "variant": [4, 5, 7, 8], "laptop": [4, 5], "desktop": [4, 5, 7], "enterpris": [4, 5, 7, 8, 9], "ceil": 4, "notabl": [4, 5, 8, 9], "bitnet": 4, "cpp": [4, 9], "arm": 4, "x86": 4, "speedup": [4, 7], "37x": 4, "07x": 4, "17x": 4, "beyond": [4, 5, 8], "raw": [4, 5, 7, 8, 9], "speed": [4, 5, 7, 8], "energi": [4, 5, 6], "55": [4, 5, 6, 7], "70": [4, 5, 7], "71": [4, 5], "82": [4, 8], "impress": [4, 7, 9], "100b": 4, "b1": 4, "58": [4, 6, 7], "pace": [4, 5, 6, 8], "second": [4, 5, 6, 7, 8], "kernel": 4, "characterist": [4, 5, 7, 8, 9], "excit": [4, 7], "frontier": [4, 8], "compel": [4, 5, 7, 9], "acceler": [4, 5, 7, 8], "faster": [4, 7], "arithmet": [4, 5], "benefici": [4, 5, 7], "sustain": [4, 5, 6, 7, 8], "Be": [4, 5, 7, 8], "fine": [4, 5, 8], "pure": [4, 5, 7], "unlock": [4, 9], "track": [4, 5, 6, 8], "chargeback": 4, "regularli": [4, 5], "wz": 4, "jinheng": 4, "hansong": 4, "ting": [4, 8], "shaoguang": 4, "shume": 4, "ma": [4, 5, 8], "hongyu": [4, 5], "xia": [4, 5, 7], "infra": 4, "fast": [4, 5, 7, 8, 9], "lossless": 4, "16144": 4, "andreessenhorowitz24": 4, "huggingface4w": [4, 7], "2024w": [4, 7], "unsloth24": 4, "jonathan": [4, 5, 8], "ceo": [4, 5], "groq": [4, 7], "streamlin": [4, 5, 6, 7, 9], "notat": 4, "width": [4, 7], "_k": 4, "_0": 4, "matter": [5, 6], "beauti": 5, "smart": [5, 8], "agre": 5, "wrong": 5, "feynman": 5, "advent": 5, "verif": [5, 6, 7, 9], "norm": 5, "realm": 5, "convent": [5, 8], "evolut": [5, 7], "conceiv": 5, "entrench": 5, "seem": 5, "daunt": [5, 6], "ignor": 5, "outdat": [5, 6, 8, 9], "inevit": 5, "setback": 5, "imper": 5, "embrac": 5, "proactiv": [5, 8], "mindset": 5, "front": [5, 7], "incorpor": [5, 6, 7, 8, 9], "produc": [5, 6, 7, 8, 9], "novel": [5, 7], "ident": [5, 6], "isn": [5, 8], "bug": 5, "random": [5, 8, 9], "testabl": 5, "guarante": [5, 6, 7, 8, 9], "exceedingli": 5, "primari": [5, 8], "nucleu": 5, "2020": 5, "summari": [5, 6, 7, 8, 9], "alter": 5, "rigid": 5, "wildli": 5, "incoher": 5, "inadequ": [5, 8], "temp": 5, "df_result": 5, "ntemperatur": 5, "40": [5, 6, 7], "temp_respons": 5, "iterrow": [5, 8], "10000": [5, 6, 9], "appl": [5, 6, 9], "txt": [5, 6, 7, 9], "sec_fil": [5, 9], "nsecur": 5, "AND": [5, 9], "exchang": [5, 6, 8, 9], "commiss": [5, 6, 8, 9], "nwashington": 5, "20549": 5, "nform": 5, "pursuant": 5, "TO": [5, 8], "13": [5, 6, 7, 8], "OR": 5, "OF": [5, 8], "THE": [5, 8], "1934": 5, "nfor": 5, "fiscal": [5, 6], "septemb": [5, 6], "28": [5, 6, 7, 8], "nor": 5, "period": [5, 6, 8], "ncommiss": 5, "001": [5, 7], "36743": 5, "ng66145g66i43": 5, "jpg": 5, "nappl": 5, "exact": [5, 7, 8], "registr": 5, "charter": 5, "ncalifornia": 5, "t94": 5, "2404110": 5, "jurisdict": 5, "nof": 5, "employ": 5, "park": 5, "ncupertino": 5, "california": [5, 8, 9], "n95014": 5, "princip": 5, "offic": [5, 6, 8], "408": 5, "996": 5, "1010": 5, "telephon": 5, "regist": 5, "ntitl": 5, "ttrade": 5, "symbol": 5, "tname": 5, "ncommon": 5, "stock": [5, 9], "00001": 5, "naapl": 5, "tthe": 5, "nasdaq": [5, 6, 9], "llc": [5, 9], "n0": 5, "000": [5, 7, 9], "note": [5, 6, 7, 9], "2025": [5, 6], "875": 5, "625": 5, "2026": 5, "2027": 5, "375": 5, "2029": 5, "050": 5, "2031": [5, 8], "600": 5, "2042": 5, "nindic": 5, "issuer": 5, "405": 5, "nye": 5, "preced": [5, 9], "shorter": 5, "past": [5, 6, 8], "90": [5, 6, 7, 8], "submit": [5, 7, 8], "electron": 5, "232": 5, "filer": 5, "12b": [5, 8], "nlarg": 5, "tacceler": 5, "nnon": 5, "tsmaller": 5, "nemerg": 5, "nif": 5, "elect": [5, 8], "revis": [5, 8], "attest": 5, "404": 5, "sarban": 5, "oxlei": 5, "7262": 5, "firm": [5, 8], "prepar": [5, 6, 7, 8], "correct": [5, 6, 8], "restat": 5, "incent": 5, "compens": 5, "240": 5, "10d": 5, "shell": 5, "aggreg": [5, 8], "vote": 5, "held": [5, 9], "affili": [5, 9], "29": [5, 7, 8, 9], "last": [5, 6, 8, 9], "quarter": 5, "628": [5, 9], "553": [5, 9], "sole": [5, 8], "disclosur": [5, 6, 7, 8], "director": [5, 7, 8], "date": [5, 6], "exclud": 5, "n15": 5, "115": [5, 9], "823": [5, 9], "outstand": [5, 9], "octob": [5, 9], "18": [5, 6, 7, 8, 9], "ndocument": 5, "BY": 5, "nportion": 5, "meet": [5, 6, 8, 9], "sharehold": [5, 6], "iii": 5, "ntabl": 5, "npage": 5, "npart": 5, "nitem": 5, "nbusi": 5, "1a": 5, "nrisk": 5, "1b": [5, 7, 8], "nunresolv": 5, "staff": 5, "comment": 5, "n17": 5, "1c": 5, "ncybersecur": 5, "nproperti": 5, "n18": 5, "nlegal": 5, "proceed": [5, 6, 8], "nmine": 5, "ii": [5, 7, 9], "nmarket": 5, "stockhold": 5, "purchas": [5, 6, 8], "n19": 5, "reserv": [5, 6], "n20": 5, "nmanag": 5, "n21": 5, "7a": 5, "nquantit": 5, "n27": 5, "nfinanci": 5, "supplementari": 5, "n28": 5, "nchang": 5, "disagr": 5, "n51": 5, "9a": 5, "ncontrol": 5, "procedur": [5, 6, 8], "9b": 5, "nother": 5, "n52": 5, "9c": 5, "ndisclosur": 5, "foreign": [5, 6], "ndirector": 5, "corpor": [5, 6, 8], "nexecut": 5, "ownership": [5, 7], "certain": [5, 6, 8, 9], "owner": 5, "ncertain": 5, "nprincip": 5, "fee": [5, 6], "iv": 5, "nexhibit": 5, "n53": 5, "n56": 5, "nthi": 5, "litig": [5, 6, 7], "reform": 5, "1995": 5, "uncertainti": [5, 6, 7, 8], "macroeconom": [5, 6], "anticip": [5, 6, 8], "intend": [5, 7, 8], "caus": [5, 8, 9], "oblig": [5, 6], "nunless": 5, "herein": 5, "calendar": 5, "wholli": 5, "subsidiari": 5, "unless": [5, 7], "ncompani": 5, "manufactur": 5, "tablet": [5, 6, 7], "wearabl": 5, "accessori": 5, "sell": [5, 8], "varieti": [5, 7], "52": [5, 8], "53": [5, 6, 8], "week": 5, "saturdai": 5, "nproduct": 5, "niphon": 5, "io": [5, 6, 9], "iphon": [5, 6], "se": [5, 8], "nmac": 5, "maco": [5, 7], "mac": [5, 7], "macbook": 5, "air": 5, "imac": 5, "studio": 5, "nipad": 5, "multipurpos": 5, "ipado": 5, "ipad": 5, "nwearabl": 5, "home": [5, 6, 9], "smartwatch": 5, "wireless": 5, "headphon": 5, "spatial": 5, "watcho": 5, "watch": 5, "ultra": 5, "airpod": 5, "beat": [5, 7], "visiono": 5, "nhome": 5, "tv": 5, "tvo": 5, "homepod": 5, "fidel": [5, 9], "naccessori": 5, "brand": 5, "third": [5, 6, 7, 8], "parti": [5, 6, 7, 8], "nservic": 5, "nadvertis": 5, "advertis": 5, "licens": [5, 6], "napplecar": 5, "portfolio": [5, 6], "applecar": 5, "repair": 5, "coverag": [5, 6, 8], "accident": 5, "damag": [5, 8], "theft": [5, 8], "ncloud": 5, "ndigit": 5, "app": [5, 6, 7], "discov": [5, 7, 8], "download": [5, 7], "music": 5, "podcast": 5, "subscript": [5, 7], "arcad": 5, "sm": 5, "listen": [5, 7], "radio": 5, "station": 5, "magazin": 5, "exclus": 5, "sport": 5, "npayment": 5, "payment": 5, "credit": [5, 6], "pai": [5, 7], "cashless": 5, "nsegment": 5, "primarili": [5, 6, 8], "geograph": [5, 6, 8], "basi": [5, 7], "segment": [5, 6, 8, 9], "america": [5, 6], "europ": 5, "china": [5, 6, 7, 8], "japan": 5, "rest": [5, 7], "asia": 5, "pacif": 5, "north": [5, 8], "south": 5, "european": [5, 8], "india": 5, "middl": [5, 7, 8], "east": 5, "africa": 5, "mainland": 5, "kong": 5, "taiwan": 5, "australia": 5, "asian": [5, 6], "although": [5, 7], "partner": [5, 6, 7, 8], "mid": [5, 6], "resel": [5, 6], "retail": 5, "sale": [5, 6], "indirect": 5, "channel": [5, 8], "cellular": 5, "carrier": 5, "net": [5, 6, 9], "38": [5, 6, 7, 8], "ncompetit": 5, "downward": 5, "pressur": [5, 8], "gross": [5, 8], "cycl": [5, 8], "competitor": [5, 6, 7, 8], "compet": [5, 6, 7], "imit": 5, "infring": [5, 7], "intellectu": [5, 7, 8], "marketplac": [5, 8], "nearli": [5, 7], "reput": [5, 8], "expand": [5, 6, 7, 8], "profit": [5, 6, 8, 9], "illegitim": [5, 8], "collabor": [5, 7, 8], "nsuppli": 5, "nalthough": 5, "particip": 5, "shortag": 5, "commod": [5, 6, 7], "fluctuat": [5, 6], "commonli": 5, "until": [5, 8, 9], "supplier": 5, "matur": 5, "concentr": [5, 6], "enter": [5, 9], "agreement": [5, 6], "suppli": [5, 6, 9], "renew": [5, 6], "nresearch": 5, "nbecaus": 5, "upon": [5, 6, 8], "flow": [5, 6, 9], "acquisit": [5, 6, 8], "nintellectu": 5, "broad": [5, 6, 7, 9], "patent": 5, "copyright": [5, 7], "trademark": 5, "secret": 5, "differenti": 5, "skill": [5, 8], "personnel": 5, "pursu": [5, 8], "thousand": [5, 7], "durat": 5, "adequ": [5, 8], "nin": 5, "holidai": [5, 8], "fill": 5, "inventori": 5, "older": [5, 7], "newer": 5, "distributor": 5, "nhuman": 5, "strive": 5, "retain": [5, 6, 7, 8], "talent": [5, 6], "member": [5, 8], "164": 5, "ncompens": 5, "equit": 5, "succe": 5, "health": [5, 6, 8], "awai": [5, 6, 8], "ngrowth": 5, "career": 5, "leadership": [5, 8], "nworkplac": 5, "workplac": 5, "ninclus": 5, "workforc": 5, "nengag": 5, "among": [5, 7, 8, 9], "everyon": [5, 7], "gaug": 5, "sentiment": [5, 6, 7, 9], "nhealth": 5, "everywher": 5, "crisi": 5, "visitor": 5, "navail": 5, "quarterli": 5, "q": [5, 6, 7, 8], "amend": 5, "sec": [5, 6, 9], "Such": [5, 8], "charg": 5, "investor": [5, 6, 9], "aspx": 5, "websit": [5, 7, 8], "environment": [5, 8], "referenc": [5, 6], "inact": 5, "textual": 5, "unknown": [5, 6, 8], "advers": 5, "conjunct": 5, "consolid": [5, 6], "nmacroeconom": 5, "facil": 5, "assembli": 5, "site": [5, 9], "nadvers": 5, "slow": 5, "recess": 5, "unemploy": [5, 6], "inflat": [5, 6], "tighter": 5, "currenc": [5, 6], "monetari": 5, "contract": [5, 7], "logist": 5, "instabl": [5, 8], "inabl": [5, 6], "financ": [5, 7, 8], "insolv": 5, "counterparti": 5, "debt": 5, "liquid": [5, 6], "fair": [5, 8], "instrument": 5, "polit": [5, 8], "disput": 5, "geopolit": 5, "tension": [5, 8], "terror": 5, "accid": 5, "interrupt": 5, "npolit": 5, "outsourc": [5, 6], "korea": 5, "vietnam": 5, "restrict": [5, 7, 8, 9], "tariff": 5, "export": [5, 6], "portion": [5, 7], "revenu": [5, 6, 9], "restructur": 5, "ceas": 5, "escal": [5, 8], "nmani": 5, "prone": [5, 8], "earthquak": 5, "climat": 5, "weather": 5, "plant": 5, "terrorist": [5, 8], "attack": [5, 8], "hostil": 5, "ransomwar": 5, "cybersecur": [5, 6, 8], "labor": 5, "nsuch": 5, "imposs": [5, 7], "slowdown": 5, "outag": 5, "neg": [5, 6, 8, 9], "pandem": 5, "covid": 5, "economi": 5, "imposit": 5, "stringent": [5, 7, 8], "travel": 5, "freight": 5, "movement": 5, "ramp": 5, "nfollow": 5, "expenditur": 5, "resum": 5, "exacerb": 5, "insur": 5, "nglobal": 5, "unabl": 5, "assur": [5, 8], "minor": [5, 6, 8], "naddition": 5, "intensifi": 5, "seamlessli": 5, "nto": 5, "stimul": 5, "ndue": 5, "upgrad": 5, "quantiti": 5, "defect": 5, "defici": 5, "supersed": 5, "nsubstanti": 5, "transport": 5, "reimburs": 5, "warranti": 5, "unanticip": 5, "liabil": 5, "final": [5, 6, 8, 9], "finish": [5, 8], "destin": 5, "prepay": 5, "termin": [5, 7], "recover": 5, "exposur": [5, 8], "nfutur": 5, "semiconductor": 5, "suffer": [5, 8], "constrain": [5, 7, 9], "shipment": 5, "unexpectedli": 5, "interfer": 5, "unsaf": [5, 8], "expos": [5, 6, 8], "widespread": [5, 8], "vulner": [5, 6, 8], "compromis": [5, 7, 8], "claim": [5, 6, 7, 8], "intang": 5, "lost": [5, 6, 8], "cancel": 5, "obsolet": 5, "exce": [5, 8], "realiz": 5, "accru": 5, "excess": 5, "impair": 5, "whenev": 5, "circumst": 5, "amount": [5, 6, 8, 9], "carri": [5, 7, 9], "incur": 5, "unpredict": [5, 8], "obsolesc": 5, "forecast": [5, 6, 8], "incorrectli": [5, 8, 9], "extens": [5, 7, 9], "issuanc": 5, "unknowingli": [5, 8], "notifi": 5, "preclud": 5, "bui": 5, "percept": 5, "android": [5, 6], "playstat": 5, "nintendo": 5, "xbox": 5, "inclin": 5, "devot": 5, "dissatisfi": 5, "vast": [5, 8], "storefront": 5, "safari": 5, "union": [5, 8], "eu": [5, 6, 8], "dma": [5, 6], "narrow": [5, 7, 8], "scope": [5, 6, 7, 8], "elimin": [5, 7], "nfailur": 5, "appeal": [5, 6], "subscrib": [5, 6], "nsome": 5, "manner": [5, 6, 8], "nurtur": 5, "nmuch": 5, "chief": [5, 6], "silicon": 5, "vallei": 5, "constantli": 5, "driver": [5, 7], "recruit": 5, "subsidi": 5, "staf": 5, "contractor": 5, "placement": 5, "increment": 5, "weaken": 5, "telecommun": 5, "war": 5, "virus": 5, "ins": 5, "incid": [5, 8], "ineffect": 5, "thing": [5, 9], "interf": 5, "imped": 5, "ship": 5, "nloss": 5, "unauthor": [5, 8], "confidenti": [5, 7], "encrypt": 5, "But": [5, 8, 9], "behalf": 5, "normal": [5, 6, 8, 9], "investig": [5, 8], "penalti": [5, 7], "frequenc": [5, 7, 8], "actor": [5, 8], "circumv": [5, 8], "obfusc": 5, "forens": 5, "hinder": [5, 9], "recov": 5, "perpetr": 5, "profil": [5, 7], "authent": 5, "hack": [5, 8], "malfeas": 5, "faulti": 5, "password": 5, "irregular": 5, "fraudul": 5, "induc": 5, "disclos": [5, 6, 9], "usernam": 5, "turn": [5, 6, 8, 9], "multifactor": 5, "unusu": 5, "freez": 5, "suspici": 5, "nwhile": 5, "ninvest": 5, "ongo": [5, 6, 7], "contempl": 5, "endeavor": 5, "distract": 5, "tangibl": 5, "approv": 5, "oner": 5, "ventur": 5, "riski": 5, "leas": 5, "unfavor": [5, 6], "arisen": 5, "ordinari": 5, "cours": [5, 7, 8], "resolv": [5, 7, 8], "sometim": 5, "indemnif": 5, "indemnifi": 5, "alleg": 5, "magnitud": 5, "assert": [5, 6], "royalti": 5, "vigor": 5, "defend": 5, "court": [5, 7], "internation": 5, "plaintiff": 5, "injunct": 5, "relief": 5, "nregardless": 5, "merit": 5, "recognit": [5, 7, 8], "settl": 5, "uncertain": [5, 6], "disgorg": 5, "remedi": [5, 8], "worldwid": 5, "antitrust": [5, 6], "bill": [5, 6], "commerc": 5, "televis": 5, "film": 5, "anticorrupt": 5, "cash": [5, 6], "repatri": 5, "launder": 5, "tax": [5, 6], "wast": 5, "recycl": 5, "ncomplianc": 5, "impos": [5, 7, 8, 9], "agent": [5, 7, 8], "nregulatori": 5, "ban": [5, 8], "nexpect": 5, "increasingli": [5, 7, 8, 9], "greenhous": 5, "ga": 5, "emiss": 5, "civil": 5, "disagre": 5, "perceiv": 5, "feder": 5, "nfrom": 5, "noncompli": 5, "individu": [5, 6, 7, 8], "lawsuit": [5, 7], "monopol": 5, "nfurther": 5, "earn": 5, "search": [5, 6, 7, 8], "nthere": 5, "transfer": 5, "pass": [5, 6, 7, 8, 9], "pend": 5, "inquiri": [5, 8], "government": 5, "entiti": [5, 7, 8, 9], "biometr": 5, "notif": 5, "permit": [5, 7, 9], "healthcar": [5, 6, 7], "liabl": 5, "investigatori": 5, "cardhold": 5, "acquir": 5, "denomin": 5, "offset": 5, "strengthen": [5, 8], "nconvers": 5, "thu": 5, "hedg": 5, "deterior": 5, "sovereign": 5, "heighten": [5, 8], "worsen": 5, "A": [5, 6, 7, 8, 9], "collater": 5, "bank": 5, "unsecur": 5, "subassembli": 5, "assembl": 5, "legisl": 5, "ireland": [5, 8], "singapor": 5, "organis": 5, "statutori": 5, "valuat": [5, 6], "defer": 5, "bodi": [5, 8], "adequaci": 5, "ow": 5, "ngener": 5, "repurchas": 5, "dividend": 5, "consumm": 5, "declar": [5, 6], "board": [5, 6, 8], "unresolv": 5, "nnone": 5, "threat": [5, 6, 8], "postur": 5, "25": [5, 6, 7, 8], "2016": 5, "coordin": [5, 8], "committe": [5, 8], "oversight": [5, 8], "counsel": 5, "chair": 5, "headquart": 5, "cupertino": [5, 9], "center": [5, 8, 9], "formal": [5, 8, 9], "conclud": [5, 7], "uninstal": 5, "web": [5, 6, 7, 8], "browser": 5, "june": 5, "contractu": 5, "desist": 5, "stai": [5, 7], "grant": 5, "ndepart": 5, "justic": 5, "depart": [5, 8], "doj": 5, "district": 5, "attornei": 5, "jersei": 5, "redress": [5, 8], "anticompetit": 5, "nonmonetari": 5, "defens": [5, 8], "nepic": 5, "epic": 5, "northern": 5, "unfair": [5, 8], "enjoin": 5, "extern": [5, 6, 8], "januari": 5, "motion": 5, "oppos": [5, 8], "vacat": 5, "fourth": 5, "mine": 5, "nnot": 5, "aapl": 5, "nholder": 5, "na": [5, 8], "301": 5, "npurchas": 5, "nshare": 5, "nperiod": 5, "ttotal": 5, "taverag": 5, "npaid": 5, "nannounc": 5, "napproxim": 5, "That": [5, 6, 8, 9], "nunder": 5, "njune": 5, "august": [5, 6, 8], "nopen": 5, "negoti": [5, 8], "t35": 5, "697": 5, "t224": 5, "naugust": 5, "31": [5, 6, 7], "t42": 5, "910": 5, "t221": 5, "39": [5, 6, 7], "nseptemb": 5, "t33": 5, "653": 5, "t222": 5, "86": [5, 7], "ntotal": [5, 8], "t112": 5, "260": 5, "t89": 5, "074": 5, "110": 5, "10b5": 5, "reinvest": 5, "dow": 5, "supersector": 5, "27": [5, 7, 8], "2019": 5, "n2218": 5, "tseptemb": 5, "t100": 5, "t207": 5, "t273": 5, "t281": 5, "t322": 5, "t430": 5, "t113": 5, "t156": 5, "t131": 5, "t155": 5, "t210": 5, "ndow": 5, "t146": 5, "t216": 5, "t215": 5, "nfirst": 5, "nsecond": 5, "nthird": 5, "sequoia": 5, "nfourth": 5, "plu": [5, 7], "nfiscal": 5, "six": 5, "realign": 5, "span": [5, 7, 8], "indirectli": 5, "n2024": 5, "tchang": 5, "t2023": 5, "t2022": 5, "namerica": 5, "t167": 5, "045": 5, "t3": 5, "t162": 5, "560": 5, "t169": 5, "658": 5, "neurop": 5, "t101": 5, "328": 5, "t7": 5, "294": 5, "t95": 5, "118": 5, "ngreater": 5, "t66": 5, "952": 5, "t72": 5, "559": 5, "t74": 5, "njapan": 5, "t25": 5, "052": 5, "t24": 5, "257": 5, "977": 5, "nrest": 5, "t30": 5, "t4": 5, "t29": 5, "615": 5, "t1": 5, "t391": 5, "035": 5, "t2": 5, "t383": 5, "285": 5, "t394": 5, "weak": [5, 6, 8], "renminbi": 5, "yen": [5, 9], "t201": 5, "183": 5, "t200": 5, "583": 5, "t205": 5, "489": 5, "984": 5, "357": 5, "t40": 5, "177": [5, 8], "t26": 5, "694": 5, "t28": 5, "300": 5, "292": 5, "t37": 5, "005": 5, "t39": 5, "845": [5, 8], "t41": 5, "241": 5, "n96": 5, "169": 5, "t13": 5, "t85": 5, "t9": 5, "t78": 5, "129": [5, 8], "amort": 5, "bundl": 5, "flat": [5, 6], "ngross": 5, "t109": 5, "633": 5, "t108": 5, "803": 5, "t114": 5, "728": 5, "t71": 5, "t60": 5, "345": 5, "t56": 5, "054": 5, "t180": 5, "683": 5, "148": 5, "t170": 5, "782": 5, "t36": 5, "t73": 5, "t70": 5, "t46": 5, "t44": 5, "t43": 5, "noper": 5, "t31": 5, "370": 5, "t5": 5, "915": 5, "t14": 5, "251": 5, "npercentag": 5, "t8": 5, "nsell": 5, "administr": 5, "097": 5, "932": 5, "094": 5, "t6": 5, "t57": 5, "467": 5, "t54": 5, "847": 5, "t51": 5, "t15": 5, "headcount": 5, "nprovis": 5, "749": 5, "t16": 5, "741": 5, "t19": 5, "neffect": 5, "nstatutori": 5, "t21": 5, "aid": [5, 8], "nliquid": 5, "unrestrict": 5, "140": 5, "ndebt": 5, "97": [5, 6, 8], "payabl": 5, "promissori": 5, "nleas": 5, "nmanufactur": 5, "noncancel": 5, "ndeem": 5, "tcja": 5, "nstate": 5, "fund": [5, 6, 7], "escrow": 5, "ncapit": 5, "95": [5, 8], "nrecent": 5, "pronounc": 5, "nincom": 5, "fasb": 5, "asu": 5, "09": [5, 6, 8], "740": 5, "reconcili": [5, 6], "reconcil": [5, 9], "disaggreg": 5, "prospect": 5, "novemb": [5, 8], "07": [5, 6, 8, 9], "280": 5, "maker": 5, "codm": 5, "retrospect": 5, "ncritic": 5, "conform": [5, 9], "gaap": 5, "nuncertain": 5, "domest": 5, "taxat": 5, "resolut": [5, 6], "conting": 5, "ninterest": 5, "forth": 5, "hypothet": 5, "nsensit": 5, "nhypothet": 5, "nrate": 5, "npotenti": 5, "n100": 5, "tenor": 5, "ndeclin": 5, "755": 5, "089": 5, "nterm": 5, "nincreas": 5, "t139": 5, "t194": 5, "nforeign": 5, "var": 5, "mont": 5, "carlo": 5, "interv": 5, "538": 5, "669": 5, "nindex": 5, "tpage": 5, "nconsolid": 5, "n29": 5, "n30": 5, "sheet": 5, "n31": 5, "n32": 5, "n33": 5, "nnote": 5, "n34": 5, "nreport": 5, "n48": 5, "nall": 5, "omit": 5, "submiss": 5, "nyear": 5, "n2023": 5, "n2022": 5, "nnet": 5, "t294": 5, "866": 5, "t298": 5, "085": 5, "t316": 5, "199": 5, "t96": 5, "ncost": 5, "t185": 5, "233": 5, "t189": 5, "282": 5, "471": 5, "119": 5, "855": 5, "t22": 5, "075": 5, "352": 5, "t214": 5, "137": 5, "t223": 5, "546": 5, "t123": 5, "216": 5, "t119": 5, "437": 5, "t269": 5, "565": 5, "334": 5, "485": 5, "736": 5, "103": 5, "t93": 5, "995": 5, "t99": 5, "nearn": 5, "nbasic": 5, "ndilut": 5, "08": [5, 7, 9], "343": [5, 8], "783": 5, "744": 5, "215": 5, "963": 5, "095": 5, "812": 5, "547": 5, "325": 5, "819": 5, "nsee": 5, "translat": [5, 7, 8], "t395": 5, "765": 5, "511": 5, "unreal": 5, "832": 5, "t323": 5, "212": 5, "nadjust": 5, "337": 5, "717": 5, "394": 5, "138": 5, "850": 5, "563": 5, "104": 5, "t204": 5, "t253": 5, "816": 5, "899": 5, "272": 5, "t98": 5, "016": 5, "652": 5, "t88": 5, "531": 5, "nasset": 5, "ncurrent": 5, "ncash": 5, "943": 5, "965": 5, "228": 5, "590": 5, "naccount": 5, "410": 5, "508": 5, "nvendor": 5, "t32": 5, "833": 5, "477": 5, "ninventori": 5, "286": 5, "331": 5, "287": 5, "695": 5, "t152": 5, "987": 5, "t143": 5, "566": 5, "t91": 5, "479": 5, "544": 5, "t45": 5, "680": 5, "715": 5, "834": 5, "t64": 5, "758": 5, "t211": 5, "993": 5, "t209": 5, "017": 5, "t364": 5, "980": [5, 8], "t352": 5, "nliabil": 5, "t68": 5, "960": 5, "t62": 5, "611": 5, "304": 5, "t58": 5, "829": 5, "ndefer": 5, "249": 5, "061": 5, "ncommerci": 5, "967": 5, "985": 5, "t10": 5, "912": 5, "822": 5, "t176": 5, "392": 5, "t145": 5, "308": 5, "750": 5, "888": 5, "t49": 5, "848": 5, "638": 5, "t308": 5, "030": [5, 7], "t290": 5, "ncommit": 5, "nsharehold": 5, "400": [5, 6], "116": 5, "786": 5, "550": 5, "n83": 5, "276": 5, "naccumul": 5, "deficit": 5, "154": 5, "214": 5, "172": 5, "452": 5, "950": 5, "146": [5, 8], "t50": 5, "672": 5, "t63": 5, "090": 5, "nbegin": 5, "849": 5, "365": 5, "423": 5, "346": [5, 6], "175": 5, "withheld": 5, "settlement": 5, "521": 5, "971": 5, "t12": 5, "034": 5, "t11": 5, "nend": 5, "t83": 5, "nretain": 5, "068": 5, "562": 5, "ndividend": 5, "218": 5, "793": 5, "612": 5, "099": 5, "454": 5, "846": 5, "77": [5, 6, 7], "046": 5, "186": 5, "109": 5, "t163": 5, "rsu": 5, "t0": 5, "98": [5, 6, 7], "94": [5, 6, 7, 8], "737": 5, "929": 5, "ndepreci": 5, "445": 5, "519": 5, "688": 5, "038": 5, "266": 5, "227": 5, "006": 5, "788": 5, "356": 5, "271": 5, "520": 5, "618": 5, "484": 5, "731": 5, "684": 5, "499": 5, "020": 5, "889": 5, "448": 5, "552": 5, "031": 5, "t118": 5, "254": 5, "t110": 5, "543": 5, "t122": 5, "151": 5, "48": [5, 7], "656": 5, "513": 5, "76": [5, 8], "923": 5, "nproce": 5, "211": 5, "686": 5, "917": 5, "135": 5, "828": [5, 6], "446": 5, "447": 5, "959": 5, "708": 5, "086": 5, "935": 5, "705": 5, "354": 5, "nfinanc": 5, "441": 5, "431": 5, "223": [5, 8], "234": [5, 8], "025": 5, "841": 5, "nrepurchas": 5, "949": 5, "89": [5, 8], "402": 5, "465": 5, "nrepay": 5, "958": 5, "repay": 5, "978": 5, "955": 5, "361": 5, "581": 5, "160": 5, "121": 5, "983": 5, "488": 5, "794": 5, "760": 5, "nsupplement": 5, "102": 5, "t18": 5, "679": 5, "573": 5, "33": [5, 6, 7, 8], "nbasi": 5, "prior": [5, 8], "reclassifi": 5, "nrevenu": 5, "remit": [5, 8], "straight": 5, "vest": 5, "sold": 5, "nderiv": 5, "nonleas": 5, "34": [5, 6, 8], "entitl": 5, "commenc": 5, "deliveri": 5, "stand": 5, "ssp": 5, "icloud": 5, "siri": 5, "discount": 5, "undeliv": 5, "unbil": 5, "n26": 5, "n37": 5, "moder": [5, 7], "64": [5, 7, 8], "dilut": 5, "nnumer": 5, "ndenomin": 5, "nweight": 5, "312": 5, "316": 5, "856": 5, "antidilut": 5, "tunreal": 5, "ngain": 5, "tfair": 5, "nvalu": 5, "tcash": 5, "nequival": 5, "tcurrent": 5, "tnon": 5, "t27": 5, "nlevel": 5, "nmonei": 5, "t778": 5, "nmutual": 5, "n515": 5, "t105": 5, "t617": 5, "nsubtot": 5, "293": 5, "395": 5, "nu": 5, "treasuri": 5, "516": 5, "t212": 5, "087": 5, "380": 5, "159": 5, "t703": 5, "t17": 5, "568": 5, "158": 5, "810": 5, "ncertif": 5, "deposit": 5, "t873": 5, "t387": 5, "t478": 5, "066": 5, "ncorpor": 5, "t65": 5, "622": 5, "t270": 5, "953": 5, "939": 5, "027": 5, "t47": 5, "886": 5, "nmunicip": 5, "t412": 5, "t405": 5, "t190": 5, "nmortgag": 5, "595": 5, "t175": 5, "403": 5, "t23": 5, "367": 5, "278": [5, 8], "t132": 5, "t583": 5, "635": 5, "t128": 5, "056": 5, "966": 5, "t34": 5, "t160": 5, "t688": 5, "650": 5, "36": [5, 6, 7, 8], "359": [5, 8], "t481": 5, "n442": 5, "t428": 5, "t923": 5, "t909": 5, "406": 5, "114": 5, "468": 5, "136": 5, "t271": 5, "533": 5, "048": [5, 7], "491": 5, "332": 5, "t320": 5, "t608": 5, "t76": 5, "840": 5, "956": 5, "890": 5, "t20": 5, "627": 5, "243": 5, "t628": 5, "t602": 5, "t192": 5, "t410": 5, "735": 5, "636": 5, "t344": 5, "t144": 5, "470": 5, "657": 5, "831": 5, "125": 5, "162": 5, "t173": 5, "752": 5, "corrobor": 5, "mortgag": [5, 6], "classifi": [5, 8], "37": [5, 7, 8], "swap": 5, "remeasur": 5, "notion": 5, "069": 5, "730": 5, "575": 5, "493": 5, "t104": 5, "777": 5, "nhedg": 5, "433": 5, "505": 5, "247": [5, 8], "ntrade": 5, "41": [5, 7, 8], "44": [5, 8], "depreci": 5, "nland": 5, "690": 5, "nmachineri": 5, "t80": 5, "205": [5, 7], "314": 5, "nleasehold": 5, "839": 5, "599": 5, "73": [5, 7, 8], "884": 5, "852": 5, "t55": 5, "906": 5, "601": 5, "703": 5, "010": 5, "457": 5, "634": 5, "391": 5, "neuropean": 5, "opinion": [5, 6, 8], "1991": 5, "2007": 5, "irish": 5, "branch": 5, "2003": 5, "2014": 5, "2015": 5, "minist": 5, "juli": [5, 8], "annul": 5, "ecj": 5, "hear": 5, "asid": 5, "confirm": 5, "unrecogn": [5, 6], "nfeder": 5, "571": 5, "080": 5, "644": 5, "265": 5, "801": 5, "726": 5, "570": 5, "298": 5, "49": [5, 6, 8], "t84": 5, "428": 5, "603": 5, "483": [5, 8], "t347": 5, "t669": 5, "076": 5, "830": 5, "419": 5, "072": 5, "pretax": 5, "72": [5, 6, 8], "ncomput": 5, "885": 5, "012": 5, "124": 5, "518": 5, "nimpact": 5, "246": 5, "311": 5, "366": 5, "397": 5, "nexcess": 5, "893": 5, "871": 5, "192": [5, 8], "739": 5, "ntax": 5, "carryforward": 5, "302": 5, "naccru": 5, "413": [5, 8], "421": 5, "nunreal": 5, "173": 5, "168": 5, "873": 5, "743": 5, "nless": 5, "374": 5, "007": 5, "369": 5, "551": 5, "998": 5, "nright": 5, "179": 5, "nminimum": 5, "674": 5, "940": 5, "t511": 5, "t455": 5, "t490": 5, "805": 5, "202": 5, "indefinit": 5, "temporari": 5, "727": 5, "044": 5, "284": 5, "ndecreas": 5, "386": 5, "463": 5, "982": 5, "542": 5, "936": 5, "070": 5, "expir": 5, "statut": 5, "229": 5, "494": 5, "closur": 5, "intercompani": 5, "exceed": [5, 8], "multiyear": 5, "exercis": 5, "noncash": 5, "rou": 5, "tfinanci": 5, "t2024": 5, "tother": 5, "661": 5, "tproperti": 5, "015": 5, "303": 5, "676": 5, "t165": 5, "t752": 5, "t859": 5, "430": 5, "842": [5, 8], "tfinanc": 5, "n2025": 5, "820": 5, "t171": 5, "991": 5, "n2026": 5, "914": 5, "n2027": 5, "t59": 5, "733": 5, "n2028": 5, "360": 5, "t38": 5, "398": 5, "n2029": 5, "187": 5, "nthereaft": 5, "t837": 5, "undiscount": 5, "790": 5, "imput": 5, "376": 5, "534": 5, "t896": 5, "borrow": 5, "proce": 5, "nine": [5, 8], "nmatur": 5, "333": 5, "264": 5, "948": 5, "645": 5, "309": 5, "arrear": 5, "namount": 5, "n2013": 5, "nfix": 5, "2062": 5, "t97": 5, "341": 5, "03": [5, 6], "65": [5, 8], "t106": 5, "572": 5, "n97": 5, "nunamort": 5, "321": 5, "358": 5, "113": 5, "662": 5, "930": 5, "342": 5, "800": 5, "180": 5, "88": [5, 6], "ndure": 5, "425": 5, "426": 5, "372": 5, "589": 5, "055": 5, "appreci": 5, "four": [5, 7, 8], "holder": [5, 7], "n2014": 5, "bonu": 5, "nrestrict": 5, "nnumber": 5, "nrsu": 5, "ngrant": 5, "naggreg": 5, "nfair": 5, "nbalanc": 5, "t240": 5, "427": [5, 8], "t75": 5, "t150": 5, "861": 5, "501": 5, "768": 5, "87": [5, 6, 7, 8], "101": [5, 8], "878": 5, "144": 5, "t127": 5, "t135": 5, "91": [5, 8], "456": 5, "78": [5, 7, 8], "59": [5, 8], "t140": 5, "326": 5, "t158": 5, "204": 5, "350": 5, "002": [5, 7], "nuncondit": 5, "uncondit": 5, "206": 5, "440": 5, "156": 5, "t633": 5, "t670": 5, "226": 5, "45": 5, "nconting": 5, "accrual": 5, "nconcentr": 5, "attribut": [5, 6, 7, 8, 9], "46": 5, "t67": 5, "098": 5, "082": 5, "062": 5, "569": 5, "895": 5, "458": 5, "207": 5, "nonrecur": 5, "t142": 5, "196": 5, "t138": 5, "t147": 5, "859": 5, "nchina": 5, "n66": 5, "t181": 5, "887": 5, "t172": 5, "269": 5, "nlong": 5, "664": 5, "797": 5, "778": 5, "219": 5, "nopinion": 5, "nwe": 5, "fairli": 5, "pcaob": 5, "sponsor": 5, "treadwai": 5, "2013": 5, "unqualifi": [5, 6], "thereon": 5, "nthese": 5, "misstat": 5, "fraud": [5, 8], "ndescript": 5, "naudit": 5, "nhow": 5, "nmatter": 5, "qualifi": 5, "letter": [5, 6], "advisor": 5, "ernst": 5, "llp": 5, "auditor": [5, 6], "2009": 5, "nsan": 5, "jose": 5, "nnovemb": 5, "coso": 5, "nour": 5, "ndefinit": 5, "disposit": 5, "receipt": 5, "nevalu": 5, "nbase": 5, "supervis": [5, 7, 8, 9], "13a": 5, "15d": 5, "ninher": 5, "paragraph": 5, "51": [5, 8, 9], "ninsid": 5, "deirdr": 5, "brien": 5, "vice": 5, "presid": 5, "affirm": 5, "april": 5, "withhold": 5, "remitt": 5, "mr": 5, "copi": [5, 6], "solicit": 5, "00042": 5, "nincorpor": 5, "texhibit": 5, "descript": [5, 6, 7, 8, 9], "tform": 5, "tfile": 5, "nrestat": 5, "namend": 5, "bylaw": 5, "nindentur": 5, "york": [5, 6, 7, 9], "mellon": 5, "truste": 5, "noffic": 5, "certif": 5, "2018": 5, "85": [5, 7, 8], "05": [5, 6], "2044": 5, "februari": 5, "2045": 5, "900": 5, "700": [5, 7], "250": [5, 8], "2036": 5, "2046": 5, "450": 5, "2047": 5, "2049": 5, "2030": 5, "2050": 5, "2060": 5, "2028": 5, "2041": 5, "2061": 5, "2032": 5, "2052": 5, "54": [5, 6], "2033": 5, "2053": 5, "n12": 5, "nsubsidiari": 5, "n23": 5, "nconsent": 5, "n24": 5, "npower": 5, "signatur": 5, "nrule": 5, "nsection": 5, "1350": 5, "n101": 5, "ninlin": 5, "xbrl": 5, "n104": 5, "inlin": 5, "compensatori": 5, "herewith": 5, "furnish": 5, "herebi": 5, "undertak": 5, "56": [5, 7, 8], "nsignatur": 5, "npursuant": 5, "duli": 5, "undersign": 5, "thereunto": 5, "ndate": 5, "nby": 5, "luca": [5, 9], "maestri": 5, "nluca": 5, "nsenior": 5, "nchief": 5, "nknow": 5, "THESE": 5, "appoint": 5, "cook": 5, "jointli": 5, "her": 5, "substitut": 5, "him": 5, "thereto": 5, "therewith": 5, "ratifi": 5, "virtu": 5, "hereof": 5, "nname": 5, "ttitl": 5, "tdate": 5, "tchief": 5, "tnovemb": 5, "ntimothi": 5, "tsenior": 5, "kondo": 5, "nchri": 5, "wanda": 5, "austin": 5, "nwanda": 5, "gorski": 5, "tdirector": 5, "nalex": 5, "jung": 5, "nandrea": 5, "arthur": 5, "levinson": 5, "narthur": 5, "monica": 5, "lozano": 5, "nmonica": 5, "ronald": 5, "sugar": 5, "nronald": 5, "susan": 5, "wagner": 5, "nsusan": 5, "57": [5, 7], "turbo": [5, 7, 9], "outlin": [5, 7, 8], "invdestacksmeticsisdict": 5, "setispect": 5, "20cyan": 5, "evaluationseld": 5, "anvis": 5, "droitent": 5, "discernminerv": 5, "versbobprefvers": 5, "vo\u8be5": 5, "option\u548c": 5, "meio": 5, "\u0432\u0440\u0435\u043ccisco": 5, "dellaischenpoihscap": 5, "geme": 5, "gettim": 5, "unscal": 5, "vocabulari": [5, 7, 9], "closer": 5, "sharpen": 5, "uniform": 5, "raschka": 5, "repetit": [5, 9], "radic": 5, "grappl": 5, "safer": [5, 8], "fascin": 5, "spontan": 5, "answer": [5, 6, 7, 8, 9], "aren": [5, 7], "linear": 5, "absent": [5, 8], "coax": 5, "journei": 5, "suddenli": 5, "manifest": 5, "deliber": [5, 8], "contend": 5, "rethink": 5, "tutor": 5, "children": [5, 8], "verifi": [5, 6, 7, 9], "predefin": [5, 9], "weren": 5, "kind": [5, 6], "usual": [5, 9], "quantif": 5, "contamin": [5, 8], "unseen": [5, 8], "longitudin": 5, "mostli": [5, 9], "latter": 5, "tailor": [5, 8], "great": [5, 7, 9], "cognit": 5, "misinform": [5, 8], "fabric": [5, 8], "citat": 5, "tempor": [5, 6], "disclaim": 5, "referr": 5, "incorrect": [5, 8], "demograph": [5, 8], "stereotyp": [5, 8], "societ": [5, 8], "pii": [5, 8], "anonym": 5, "leakag": [5, 8], "carryov": 5, "fallaci": 5, "think": [5, 7, 8], "idiom": 5, "sarcasm": 5, "terminologi": 5, "lingual": 5, "misunderstand": 5, "syntax": 5, "scan": [5, 6], "compat": [5, 6, 7, 9], "overconfid": [5, 6], "clariti": [5, 6, 8, 9], "audienc": 5, "densiti": 5, "satisfact": [5, 9], "misus": [5, 8], "moral": 5, "co2": 5, "etc": [5, 6, 9], "palm": [5, 7], "easi": [5, 6, 7, 8], "synthet": [5, 7, 8, 9], "templat": [5, 6, 9], "timeout": 5, "inter": 5, "rater": 5, "ti": 5, "holist": [5, 8], "built": [5, 7, 8, 9], "experiment": [5, 7, 9], "vi": 5, "categor": [5, 7, 8, 9], "intrins": [5, 7], "extrins": 5, "sequenc": [5, 6, 7, 9], "perplex": [5, 7], "downstream": [5, 9], "synthesi": 5, "discret": 5, "prefix": [5, 8], "roug": 5, "bleu": 5, "bilingu": 5, "understudi": 5, "overlap": [5, 6], "favor": [5, 7, 9], "breviti": 5, "insensit": 5, "semant": [5, 6, 9], "orient": [5, 8], "gist": 5, "meteor": 5, "synonym": 5, "paraphras": 5, "alongsid": [5, 8], "computation": [5, 6], "cider": 5, "consensu": 5, "tf": 5, "idf": 5, "caption": 5, "reliant": [5, 6], "corpu": [5, 6, 7], "ter": 5, "edit": [5, 8], "hypothesi": 5, "penal": 5, "bertscor": 5, "contextu": [5, 8], "bert": 5, "spice": 5, "proposit": [5, 7], "scene": [5, 6, 8], "analyst": [5, 6], "rouge_1": 5, "rouge_2": 5, "ideal": [5, 7, 8, 9], "setup": [5, 7, 8, 9], "evaluate_summari": 5, "unigram": 5, "bigram": 5, "absl": 5, "py": [5, 9], "rouge_scor": 5, "generated_summari": 5, "reference_summari": 5, "google_bleu": 5, "bleu_scor": 5, "rouge1": 5, "rouge2": 5, "arbitrari": 5, "chosen": [5, 8], "sentence1": 5, "cat": [5, 8], "sat": 5, "mat": 5, "sentence2": 5, "ate": 5, "3333333333333333": 5, "7272727272727272": 5, "4444444444444445": 5, "generate_summari": 5, "summir": 5, "liner": 5, "evaluate_summary_model": 5, "model_benchmark": 5, "models_test": 5, "benchmark_summari": 5, "model_summari": 5, "evaluation_result": 5, "statu": 5, "concis": [5, 7], "element": [5, 8, 9], "verbos": [5, 7, 8, 9], "peripher": 5, "quit": [5, 6, 7, 9], "convei": 5, "breadth": 5, "Of": [5, 7, 8], "vibe": 5, "visualize_prompt_comparison": 5, "matplotlib": 5, "radar": 5, "radar_plot": 5, "tmp": 5, "ipykernel_1652501": 5, "940173201": 5, "userwarn": [5, 9], "figurecanvasagg": 5, "largest": [5, 7], "sarmah": 5, "granular": [5, 6, 7], "likert": 5, "ensembl": 5, "repeatedli": [5, 6], "fluenci": 5, "refin": 5, "integ": [5, 9], "rubric": 5, "hollist": 5, "judgeevalu": 5, "grammar": [5, 7, 9], "evaluate_with_llm": 5, "criterion": 5, "judge_model": 5, "candidate_summari": 5, "grammat": 5, "y": [5, 6, 8, 9], "z": 5, "w": [5, 6, 7, 8], "benchmark_model": 5, "test_model": 5, "input_text": [5, 6, 7], "trillion": [5, 7], "evals_list": 5, "1775618912": 5, "slightli": 5, "drift": [5, 8], "lowest": [5, 7], "firstli": 5, "overhead": [5, 7], "egocentr": 5, "tight": 5, "medicin": [5, 8], "glider": 5, "deshpand": 5, "3b": 5, "685": 5, "aplic": 5, "golden": 5, "earlier": [5, 8], "depict": [5, 8, 9], "multilingu": [5, 7, 8], "arena": 5, "randomli": 5, "customiz": [5, 7, 8], "irrelev": 5, "unhelp": [5, 8], "occasion": 5, "rare": 5, "perfectli": 5, "cater": [5, 7], "critiqu": [5, 8], "elo": 5, "exam": 5, "probe": [5, 8], "certifi": 5, "glue": 5, "entail": [5, 7], "superglu": 5, "successor": 5, "grew": 5, "big": [5, 7], "bench": [5, 7], "srivastava": 5, "truthfulqa": [5, 7], "multitask": 5, "hendryck": [5, 8], "multidisciplinari": 5, "stanford": 5, "helm": 5, "multidimension": 5, "surround": [5, 7, 8, 9], "humanev": [5, 7], "lmsy": 5, "brought": 5, "dialogu": [5, 7], "chiang": 5, "gather": 5, "hundr": [5, 7], "alpacaev": 5, "duboi": 5, "mt": 5, "argilla": 5, "mila": 5, "mit": [5, 7], "contributor": [5, 7, 9], "western": 5, "centric": 5, "divid": [5, 6, 8], "subset": [5, 8], "agnost": 5, "dialect": 5, "render": [5, 8], "crowdsourc": 5, "livebench": 5, "white": [5, 8], "resili": [5, 6, 8], "meaningfulli": 5, "satur": 5, "zebralog": 5, "grid": 5, "puzzl": 5, "brailsford": 5, "1999": 5, "lsat": 5, "hous": 5, "clue": 5, "deduct": 5, "programmat": [5, 9], "2x2": 5, "6x6": 5, "shot": [5, 8, 9], "reductio": 5, "ad": [5, 6, 7, 9], "absurdum": 5, "hard": [5, 6], "10b": 5, "counterfactu": 5, "mileston": [5, 7], "came": 5, "arc": 5, "prize": [5, 8], "chollet": 5, "mike": [5, 6, 8], "knoop": 5, "founder": 5, "zapier": 5, "fran\u00e7oi": 5, "creator": [5, 7], "kera": 5, "genuin": 5, "agi": 5, "possess": [5, 6], "elementari": 5, "novelti": 5, "interpol": 5, "synthes": 5, "fly": 5, "brute": 5, "pixel": 5, "unbeaten": 5, "win": [5, 7], "takeawai": 5, "vertic": [5, 8], "finbench": 5, "legalbench": 5, "guha": 5, "berkelei": 5, "bfcl": 5, "patil": 5, "fourrier": 5, "bespok": 5, "sdk": 5, "autoregress": 5, "sub": [5, 7], "liter": 5, "disturb": 5, "zero": [5, 7, 8, 9], "varianc": [5, 8], "yt": 5, "ut": 5, "ol": 5, "heteroscedast": 5, "regress": 5, "bivari": 5, "evaluationtrack": 5, "pipelineparamet": 5, "cache_dir": 5, "max_sampl": 5, "basemodelconfig": 5, "evaluation_track": 5, "model_config": 5, "parallelismmanag": 5, "envconfig": 5, "is_accelerate_avail": 5, "datetim": [5, 6], "timedelta": [5, 6], "initprocessgroupkwarg": 5, "create_evaluation_pipelin": 5, "float16": 5, "kwargs_handl": 5, "3000": 5, "save_detail": 5, "pipeline_param": 5, "launcher_typ": 5, "env_config": 5, "override_batch_s": 5, "use_chat_templ": 5, "trust_remote_cod": 5, "pipeline_paramet": 5, "schemat": [5, 6], "vllm": [5, 9], "tgi": 5, "num_few_shot": 5, "bar": 5, "bigbench": 5, "winogrand": 5, "hellaswag": 5, "nlp": [5, 6, 7, 8], "save_and_push_result": 5, "show_result": 5, "model_arg": 5, "send": [5, 6, 7, 8, 9], "serverless": 5, "inference_server_address": 5, "inference_server_auth": 5, "model_id": 5, "null": 5, "bash": [5, 7], "command": [5, 6, 7], "model_config_path": 5, "endpoint_model": 5, "llama3": 5, "qwen2": [5, 7, 9], "alibaba": [5, 7, 9], "5b": [5, 7, 9], "hui": [5, 7], "allal": [5, 7], "cluster": 5, "noteworthi": [5, 7], "grain": [5, 7, 9], "salt": [5, 9], "modular": 5, "offici": 5, "revisit": 5, "langchain": [5, 6], "trace": 5, "langchain_tracing_v2": 5, "langchain_api_kei": 5, "hf_evalu": 5, "langsmith_evalu": 5, "ls_client": 5, "dataset_nam": 5, "create_dataset": 5, "create_exampl": 5, "dataset_id": 5, "calculate_scor": 5, "reference_output": 5, "oai_client": 5, "xp_model_nam": 5, "lastli": 5, "run_evalu": 5, "And": [5, 6, 7, 8], "upload_result": 5, "experiment_prefix": 5, "num_repetit": 5, "386a3620": 5, "9e1cc3cb": 5, "9d6a": 5, "4356": 5, "ab34": 5, "138e0abe8be4": 5, "8741976e": 5, "5268": 5, "4b75": 5, "949f": 5, "99477dde5d64": 5, "selectedsess": 5, "b831dc1e": 5, "90bc": 5, "4ed8": 5, "8080": [5, 7], "fb42444724d6": 5, "4it": 5, "latest": [5, 6, 7, 8, 9], "tobia": [5, 9], "evaluate_modul": 5, "6fc70b7be0088120a372dfdd5d320b39b8bb3630cb8029b193941d9376e86bb0": 5, "tue": 5, "nov": [5, 7], "couldn": 5, "5it": 5, "5053784e": 5, "64445871": 5, "a53c": 5, "44b1": 5, "a422": 5, "4f49b2f9656f": 5, "69": [5, 8], "4b29f3c9": 5, "9ef7e39a": 5, "2add": 5, "410c": 5, "89f8": 5, "9f1a8b198cf1": 5, "61": [5, 8], "insert": [5, 6], "combined_df": 5, "concat": [5, 8], "ignore_index": [5, 8], "execution_tim": 5, "example_id": 5, "333333": 5, "224388": 5, "feb10f92": 5, "3167": 5, "41f3": 5, "bb1c": 5, "d271153a31a8": 5, "5b196b22": 5, "9f4c": 5, "489c": 5, "b020": 5, "7823208b42d6": 5, "348101": 5, "722464": 5, "c310f159": 5, "064a": 5, "4035": 5, "97c3": 5, "a25bbf43abc2": 5, "386076": 5, "704104": 5, "f7f24899": 5, "dd50": 5, "409e": 5, "93cc": 5, "6fb1622b60bf": 5, "443038": 5, "725059": 5, "242856d6": 5, "efb5": 5, "4101": 5, "b1cf": 5, "5805532838ac": 5, "373418": 5, "795302": 5, "ce975169": 5, "a0ab": 5, "40ce": 5, "8e32": 5, "efa28d06079d": 5, "stat": [5, 7], "groupbi": [5, 8], "agg": [5, 8], "sort": 5, "sort_valu": 5, "subplot": 5, "pyplot": 5, "plt": 5, "ax1": 5, "ax2": 5, "figsiz": 5, "2ecc71": 5, "3498db": 5, "e74c3c": 5, "bleu_mean": 5, "bleu_std": 5, "enumer": [5, 6, 8], "errorbar": 5, "yerr": 5, "fmt": 5, "markers": 5, "capsiz": 5, "set_ylabel": 5, "set_titl": 5, "set_xtick": 5, "set_xticklabel": 5, "rotat": 5, "set_ylim": 5, "bottom": 5, "legend": 5, "exec_mean": 5, "exec_std": 5, "tight_layout": 5, "ndetail": 5, "4038": 5, "0453": 5, "7815": 5, "0433": 5, "3768": 5, "0424": 5, "8343": 5, "2208": 5, "3519": 5, "0775": 5, "9122": 5, "1482": 5, "377": 5, "042": 5, "078": 5, "slower": [5, 8], "04": [5, 7], "interestingli": 5, "decoupl": 5, "reload": 5, "facilit": [5, 8], "promptfooconfig": 5, "model_comparison": 5, "pretti": [5, 8], "dump": 5, "default_flow_styl": 5, "sort_kei": 5, "prompt1": 5, "defaulttest": 5, "1000m": 5, "eval_data": 5, "latency_m": 5, "totallatencym": 5, "token_usag": 5, "tokenusag": 5, "assert_pass": 5, "assertpasscount": 5, "assert_fail": 5, "assertfailcount": 5, "prompt_token": [5, 7], "num_request": 5, "numrequest": 5, "2463": 5, "000035": 5, "3773": 5, "004620": 5, "1669": 5, "000091": 5, "1669m": 5, "highest": [5, 7, 9], "3773m": 5, "00462": 5, "promptfool": 5, "manual": [5, 6, 7, 8], "redefin": 5, "prompt_comparison": 5, "prompt2": 5, "prompt3": 5, "prompt_fil": 5, "prompt_cont": 5, "BE": 5, "again": 5, "prompt_id": 5, "promptid": 5, "gradingresult": 5, "df_raw": 5, "reset_index": [5, 8], "poorli": 5, "eas": [5, 7, 8, 9], "hf": [5, 7], "plain": [5, 6, 7], "vanilla": 5, "defi": 5, "accustom": 5, "legaci": 5, "unsustain": 5, "prd": 5, "cultiv": [5, 8], "organiz": 5, "alb": [5, 7], "loubna": [5, 7], "anton": [5, 7], "lozhkov": [5, 7], "bakouch": [5, 7], "gabriel": [5, 7, 8], "mart\u00edn": [5, 7, 8], "bl\u00e1zquez": [5, 7], "lewi": [5, 6, 7], "tunstal": [5, 7], "agust\u00edn": [5, 7], "piquer": [5, 7], "andr": [5, 6, 7], "marafioti": [5, 7], "cyril": [5, 7], "zakka": [5, 7], "leandro": [5, 7], "werra": [5, 7], "wolf": [5, 7], "are24": 5, "judgearena": 5, "bps99": 5, "salli": 5, "pott": 5, "barbara": 5, "557": [5, 8], "sciencedirect": 5, "s0377221798003646": 5, "doi": [5, 6, 8, 9], "1016": 5, "s0377": 5, "2217": 5, "00364": 5, "ctj": 5, "jerri": [5, 8], "tworek": [5, 8], "heewoo": [5, 8], "jun": [5, 8], "qime": [5, 8], "henriqu": [5, 8], "pond": [5, 8], "de": [5, 8], "oliveira": [5, 8], "pinto": [5, 8], "harri": [5, 8], "yuri": 5, "burda": 5, "greg": [5, 8], "brockman": [5, 8], "raul": [5, 8], "puri": [5, 8], "gretchen": [5, 8], "krueger": [5, 8], "petrov": [5, 8], "heidi": 5, "khlaaf": 5, "girish": [5, 8], "sastri": [5, 8], "brook": [5, 8], "chan": [5, 8], "grai": [5, 8], "ryder": [5, 8], "mikhail": [5, 8], "pavlov": [5, 8], "alethea": [5, 8], "lukasz": 5, "kaiser": [5, 8], "mohammad": [5, 8], "bavarian": [5, 8], "clemen": [5, 8], "winter": [5, 8], "philipp": 5, "tillet": [5, 8], "felip": [5, 8], "petroski": [5, 8], "dave": [5, 8], "cum": [5, 8], "plappert": 5, "fotio": 5, "chantzi": [5, 8], "barn": 5, "ariel": 5, "herbert": 5, "voss": [5, 8], "hebgen": 5, "guss": 5, "nichol": 5, "paino": [5, 8], "nikola": [5, 8], "tezak": [5, 8], "babuschkin": [5, 8], "suchir": [5, 8], "balaji": [5, 8], "shantanu": [5, 8], "jain": [5, 8], "hess": [5, 8], "carr": 5, "josh": [5, 8], "achiam": [5, 8], "vedant": 5, "misra": 5, "evan": [5, 7, 8], "morikawa": [5, 8], "matthew": 5, "knight": [5, 8], "mile": [5, 8], "brundag": [5, 8], "mira": [5, 8], "murati": [5, 8], "kati": [5, 8], "mayer": [5, 8], "bob": [5, 8, 9], "mcgrew": [5, 8], "ilya": [5, 8], "sutskev": [5, 8], "wojciech": [5, 8], "zaremba": [5, 8], "2107": 5, "03374": 5, "cz": 5, "lianmin": 5, "ying": 5, "sheng": 5, "anastasio": 5, "angelopoulo": 5, "tianl": 5, "dacheng": 5, "banghua": 5, "jordan": [5, 8], "gonzalez": 5, "ion": 5, "stoica": 5, "04132": 5, "cho24a": 5, "francoi": 5, "arcpriz": 5, "cho24b": 5, "drcw": 5, "darshan": 5, "selvan": 5, "sunitha": 5, "ravi": 5, "sky": 5, "ch": 5, "bartosz": 5, "mielczarek": 5, "anand": [5, 8], "kannappan": [5, 8], "qian": [5, 8], "14140": 5, "dglh24": 5, "yann": 5, "bal\u00e1z": 5, "galambosi": 5, "tatsunori": 5, "hashimoto": 5, "debia": 5, "04475": 5, "fac24a": 5, "wiki": [5, 9], "fac24b": 5, "fac24c": 5, "model_doc": 5, "fac24d": 5, "cookbook": 5, "llm_judg": 5, "fac24f": 5, "fhwt23": 5, "cl\u00e9mentin": 5, "nathan": 5, "habib": 5, "gnh": 5, "julian": 5, "nyarko": 5, "ho": 5, "r\u00e9": 5, "adam": [5, 8], "chilton": 5, "aditya": [5, 8], "narayana": 5, "chohla": 5, "brandon": [5, 8, 9], "waldon": 5, "rockmor": 5, "diego": 5, "zambrano": 5, "dmitri": 5, "talisman": 5, "enam": 5, "hoqu": 5, "faiz": 5, "surani": 5, "frank": [5, 8], "fagan": 5, "galit": 5, "sarfati": 5, "gregori": 5, "dickinson": 5, "haggai": 5, "porat": 5, "hegland": 5, "jessica": [5, 8], "joe": [5, 8], "nudel": 5, "joel": [5, 8], "niklau": 5, "nai": 5, "choi": 5, "margaret": [5, 7], "hagan": 5, "megan": 5, "livermor": 5, "nikon": 5, "rasumov": 5, "rahe": 5, "nil": 5, "holzenberg": 5, "noam": 5, "kolt": 5, "henderson": 5, "rehaag": 5, "sharad": 5, "shang": 5, "spencer": 5, "sunni": 5, "gandhi": 5, "zur": 5, "varun": 5, "iyer": 5, "zehua": 5, "2308": 5, "11462": 5, "hbb": 5, "collin": 5, "burn": 5, "steven": [5, 8], "basart": [5, 8], "zou": [5, 8], "manta": [5, 8], "mazeika": [5, 8], "03300": 5, "hbd": 5, "maxwel": 5, "forb": 5, "yejin": 5, "curiou": 5, "neural": [5, 9], "degener": 5, "1904": 5, "09751": 5, "hyc": [5, 7], "binyuan": [5, 7], "zeyu": [5, 7], "cui": [5, 7], "jiaxi": [5, 7], "dayiheng": [5, 7], "tianyu": [5, 7], "jiajun": [5, 7], "kai": [5, 7, 8], "dang": [5, 7], "coder": [5, 7], "preprint": [5, 7, 9], "2409": [5, 7, 8], "12186": [5, 7], "lx": 5, "zhen": 5, "xiaohan": 5, "jia": 5, "yuxuan": 5, "lai": 5, "chongyang": 5, "shuai": 5, "nlg": 5, "07103": 5, "lbl": 5, "bommasani": 5, "toni": 5, "dimitri": 5, "tsipra": 5, "dilara": 5, "soylu": 5, "michihiro": 5, "yasunaga": 5, "yian": 5, "deepak": 5, "narayanan": 5, "yuhuai": 5, "newman": 5, "binhang": 5, "bobbi": 5, "ce": 5, "christian": [5, 8], "cosgrov": 5, "acosta": 5, "nava": [5, 8], "drew": 5, "hudson": 5, "zelikman": 5, "esin": 5, "durmu": 5, "faisal": 5, "ladhak": 5, "frieda": 5, "rong": [5, 6], "ren": [5, 7], "huaxiu": 5, "yao": [5, 8, 9], "jue": 5, "keshav": 5, "santhanam": 5, "laurel": 5, "lucia": 5, "mert": 5, "yuksekgonul": 5, "mirac": 5, "suzgun": 5, "niladri": 5, "chatterji": 5, "omar": 5, "khattab": 5, "chi": [5, 9], "sang": 5, "shibani": [5, 8], "santurkar": [5, 8], "surya": 5, "icard": 5, "tianyi": 5, "vishrav": 5, "chaudhari": 5, "xuechen": 5, "yuhui": 5, "yuta": 5, "koreeda": 5, "2211": 5, "09110": 5, "lbc24": 5, "ronan": 5, "bra": 5, "allenai": 5, "lhe22": [5, 7, 8], "stephani": [5, 7, 8], "owain": [5, 7, 8], "mimic": [5, 7, 8], "falsehood": [5, 7, 8], "2109": [5, 7, 8], "07958": [5, 7, 8], "pzwg23": 5, "shishir": 5, "tianjun": 5, "xin": [5, 8], "gorilla": 5, "15334": 5, "pro24": 5, "dev": 5, "ras24": 5, "sebastian": [5, 6], "scratch": 5, "1633437166": 5, "sll": 5, "bhaskarjit": 5, "mingshu": 5, "jingrao": 5, "lyu": 5, "nathalia": 5, "castellano": 5, "pasquali": 5, "dhagash": 5, "12148": 5, "srf": 5, "shivalika": 5, "angelika": 5, "roman": [5, 8], "adelani": 5, "ngui": 5, "vila": 5, "suero": 5, "peerat": 5, "limkonchotiwat": 5, "kelli": 5, "marchisio": 5, "qi": 5, "leong": 5, "yosephin": 5, "susanto": 5, "raymond": [5, 8], "ng": [5, 8], "shayn": 5, "longpr": 5, "ko": 5, "madelin": 5, "antoin": 5, "bosselut": 5, "oh": 5, "leshem": 5, "choshen": 5, "daphn": 5, "ippolito": 5, "enzo": [5, 9], "ferrant": 5, "marzieh": 5, "fadae": 5, "beyza": 5, "ermi": 5, "sara": 5, "hooker": 5, "linguist": [5, 6, 8], "03304": 5, "srr": 5, "aarohi": 5, "abhinav": 5, "rastogi": 5, "abhishek": 5, "rao": 5, "abu": 5, "awal": 5, "shoeb": 5, "abubakar": 5, "abid": [5, 7], "fisch": 5, "santoro": 5, "gupta": 5, "adri\u00e0": 5, "garriga": 5, "alonso": 5, "agnieszka": 5, "kluska": 5, "aitor": 5, "lewkowycz": 5, "akshat": 5, "warstadt": 5, "alexand": [5, 8, 9], "kocurek": 5, "ali": [5, 8], "safaya": 5, "tazarv": 5, "aman": 5, "hussain": 5, "dsouza": 5, "ambros": 5, "slone": 5, "ameet": 5, "rahan": 5, "anantharaman": 5, "ander": 5, "andreassen": 5, "madotto": 5, "santilli": 5, "stuhlm\u00fcller": 5, "la": 5, "lampinen": 5, "angelica": 5, "anh": 5, "vuong": 5, "animesh": 5, "gottardi": 5, "antonio": 5, "norelli": 5, "anu": 5, "venkatesh": 5, "arash": 5, "gholamidavoodi": 5, "arfa": 5, "tabassum": 5, "arul": 5, "menez": 5, "arun": [5, 8], "kirubarajan": 5, "asher": 5, "mullokandov": 5, "ashish": 5, "sabharw": 5, "herrick": 5, "avia": 5, "efrat": 5, "aykut": 5, "erdem": 5, "ayla": 5, "karaka\u015f": 5, "bao": [5, 7, 8], "loe": 5, "barret": [5, 8], "zoph": [5, 8], "bart\u0142omiej": 5, "bojanowski": 5, "batuhan": 5, "\u00f6zyurt": 5, "behnam": 5, "hedayatnia": 5, "neyshabur": 5, "inden": 5, "benno": 5, "stein": 5, "berk": 5, "ekmekci": 5, "blake": 5, "howald": 5, "bryan": 5, "orinion": 5, "diao": 5, "dour": 5, "stinson": 5, "cedrick": 5, "argueta": 5, "c\u00e9sar": 5, "ferri": 5, "ram\u00edrez": 5, "chandan": 5, "charl": 5, "rathkopf": 5, "chenlin": 5, "meng": 5, "chitta": 5, "baral": 5, "chiyu": 5, "callison": 5, "burch": 5, "voigt": 5, "cindi": 5, "ramirez": 5, "clara": 5, "rivera": 5, "clemencia": 5, "siro": 5, "colin": [5, 7], "raffel": [5, 7], "courtnei": 5, "ashcraft": 5, "cristina": 5, "garbacea": 5, "damien": [5, 8], "sileo": 5, "garrett": 5, "kilman": 5, "freeman": 5, "khashabi": 5, "levi": [5, 8], "mosegu\u00ed": 5, "gonz\u00e1lez": 5, "perszyk": 5, "danqi": 5, "dar": 5, "gilboa": 5, "dohan": [5, 8], "drakard": 5, "jurgen": 5, "debajyoti": 5, "datta": 5, "deni": 5, "emelin": 5, "kleyko": 5, "deniz": 5, "yuret": 5, "derek": [5, 8], "tam": [5, 9], "dieuwk": 5, "hupk": 5, "diganta": 5, "dilyar": 5, "buzan": 5, "coelho": 5, "mollo": 5, "diyi": 5, "dylan": 5, "schrader": 5, "ekaterina": 5, "shutova": 5, "ekin": 5, "dogu": 5, "cubuk": 5, "elad": 5, "segal": 5, "eleanor": 5, "hagerman": 5, "donowai": 5, "elli": 5, "pavlick": 5, "rodola": 5, "emma": 5, "lam": 5, "chu": [5, 8], "erkut": 5, "erni": 5, "dyer": 5, "jerzak": 5, "eunic": 5, "engefu": 5, "manyasi": 5, "evgenii": 5, "zheltonozhskii": 5, "fanyu": 5, "fatemeh": 5, "siar": 5, "fernando": 5, "mart\u00ednez": 5, "plume": 5, "francesca": 5, "happ\u00e9": 5, "gaurav": 5, "genta": 5, "indra": 5, "winata": 5, "gerard": 5, "melo": 5, "germ\u00e1n": 5, "kruszewski": 5, "giambattista": [5, 8], "parascandolo": [5, 8], "giorgio": 5, "mariani": 5, "gloria": 5, "gonzalo": 5, "jaimovitch": 5, "l\u00f3pez": 5, "gregor": 5, "betz": 5, "gui": [5, 7], "gur": 5, "hana": 5, "galijasev": 5, "rashkin": 5, "hannaneh": 5, "hajishirzi": 5, "harsh": 5, "hayden": 5, "bogar": 5, "henri": [5, 8], "shevlin": 5, "hinrich": 5, "sch\u00fctze": 5, "hiromu": 5, "yakura": 5, "hongm": 5, "hugh": 5, "mee": 5, "wong": [5, 6, 8], "isaac": 5, "nobl": 5, "jaap": 5, "jumelet": 5, "geissing": 5, "jaehoon": 5, "jaim": 5, "fern\u00e1ndez": 5, "fisac": 5, "simon": 5, "koppel": 5, "koco\u0144": 5, "jana": 5, "thompson": [5, 7, 8], "janel": 5, "wingfield": 5, "jarema": 5, "radom": 5, "jascha": 5, "sohl": [5, 8], "dickstein": 5, "phang": 5, "yosinski": 5, "jekaterina": 5, "novikova": 5, "jell": 5, "bosscher": 5, "jennif": 5, "marsh": 5, "jeroen": 5, "taal": 5, "engel": 5, "jesujoba": 5, "alabi": 5, "jiam": 5, "jillian": 5, "joan": 5, "waweru": 5, "burden": 5, "bali": 5, "batcheld": 5, "berant": 5, "j\u00f6rg": 5, "frohberg": 5, "jo": 5, "rozen": 5, "orallo": 5, "boudeman": 5, "guerr": 5, "tenenbaum": 5, "joyc": 5, "chua": 5, "kanclerz": 5, "karen": 5, "livescu": 5, "karl": 5, "krauth": 5, "karthik": 5, "gopalakrishnan": 5, "katerina": 5, "ignatyeva": 5, "katja": 5, "markert": 5, "kaustubh": 5, "dhole": 5, "gimpel": 5, "omondi": 5, "kori": 5, "mathewson": 5, "kristen": 5, "chiafullo": 5, "ksenia": 5, "shkaruta": 5, "shridhar": 5, "kyle": [5, 6, 8], "mcdonel": 5, "richardson": 5, "laria": 5, "reynold": 5, "leo": [5, 8], "dugan": 5, "lianhui": 5, "lidia": 5, "contrera": 5, "ochando": 5, "morenc": 5, "moschella": 5, "luci": 5, "ludwig": 5, "schmidt": [5, 8], "luheng": 5, "olivero": 5, "col\u00f3n": 5, "metz": [5, 8], "l\u00fctfi": 5, "kerem": 5, "\u015fenel": 5, "maarten": [5, 8], "bosma": 5, "sap": [5, 8], "maartj": 5, "hoev": 5, "maheen": 5, "farooqi": 5, "manaal": 5, "faruqui": 5, "marco": 5, "baturan": 5, "marelli": 5, "maru": 5, "maria": 5, "quintana": 5, "tolkiehn": 5, "mario": [5, 8], "giulianelli": 5, "martha": 5, "potthast": 5, "leavitt": 5, "hagen": 5, "m\u00e1ty\u00e1": 5, "schubert": 5, "medina": [5, 8], "orduna": 5, "baitemirova": 5, "melodi": 5, "arnaud": 5, "melvin": 5, "mcelrath": 5, "yee": 5, "cohen": 5, "ivanitskii": 5, "starritt": 5, "strube": 5, "micha\u0142": 5, "sw\u0119drowski": 5, "michel": [5, 8], "bevilacqua": 5, "mihir": 5, "kale": 5, "cain": 5, "mime": 5, "mitch": 5, "walker": 5, "mo": 5, "tiwari": 5, "mohit": 5, "bansal": 5, "moin": 5, "aminnaseri": 5, "mor": 5, "geva": 5, "mozhdeh": 5, "gheini": 5, "mukund": 5, "varma": 5, "nanyun": 5, "peng": [5, 8], "nayeon": 5, "neta": 5, "krakov": 5, "doiron": 5, "nicol": 5, "martinez": 5, "nikita": 5, "nangia": 5, "nikla": 5, "decker": 5, "muennighoff": 5, "nitish": [5, 8], "shirish": [5, 8], "keskar": [5, 8], "niveditha": 5, "constant": 5, "fiedel": 5, "nuan": 5, "wen": [5, 6], "oliv": [5, 8], "agha": 5, "elbaghdadi": 5, "omer": 5, "moreno": 5, "casar": 5, "parth": 5, "doshi": 5, "pascal": 5, "fung": 5, "pu": 5, "vicol": 5, "pegah": 5, "alipoormolabashi": 5, "peiyuan": 5, "eckerslei": 5, "phu": 5, "mon": 5, "htut": 5, "pinyu": 5, "hwang": 5, "piotr": 5, "mi\u0142kowski": 5, "piyush": 5, "pouya": 5, "pezeshkpour": 5, "priti": 5, "oli": 5, "qiaozhu": 5, "qing": 5, "qinlang": 5, "rabin": 5, "banjad": 5, "rachel": [5, 8], "etta": 5, "rudolph": 5, "raefer": 5, "rahel": 5, "haback": 5, "ramon": 5, "risco": 5, "rapha\u00ebl": 5, "milli\u00e8r": 5, "rhythm": 5, "garg": [5, 7], "rif": 5, "saurou": 5, "riku": 5, "arakawa": 5, "robb": 5, "raymaek": 5, "rohan": 5, "sikand": 5, "novak": 5, "sitelew": 5, "lebra": 5, "rosann": 5, "rowan": [5, 8], "ruslan": 5, "salakhutdinov": 5, "stoval": 5, "teehan": 5, "sahib": 5, "saif": 5, "sajant": 5, "dillav": 5, "shleifer": 5, "wiseman": 5, "gruetter": 5, "schoenholz": 5, "sanghyun": 5, "sanjeev": 5, "kwatra": 5, "sarik": 5, "ghazarian": 5, "sayan": 5, "casei": [5, 8], "bischoff": 5, "gehrmann": 5, "schuster": 5, "sepideh": 5, "sadeghi": 5, "shadi": 5, "hamdan": 5, "sharon": 5, "shashank": 5, "sherri": 5, "shi": 5, "shikhar": 5, "shima": 5, "asaadi": 5, "shubh": 5, "pachchigar": 5, "shubham": 5, "toshniw": 5, "shyam": [5, 8], "upadhyai": 5, "shyamolima": 5, "debnath": 5, "siamak": 5, "shakeri": 5, "thormey": 5, "melzi": 5, "siva": 5, "reddi": 5, "sneha": 5, "priscilla": 5, "makini": 5, "soo": 5, "hwan": 5, "toren": 5, "sriharsha": 5, "hatwar": 5, "stanisla": 5, "dehaen": 5, "stefan": 5, "divic": 5, "stella": 5, "biderman": 5, "stephen": 5, "prasad": 5, "piantadosi": 5, "stuart": [5, 8], "shieber": 5, "summer": [5, 8], "misherghi": 5, "svetlana": 5, "kiritchenko": 5, "swaroop": 5, "tal": 5, "linzen": 5, "tariq": 5, "tatsu": 5, "te": 5, "th\u00e9o": 5, "desbord": 5, "theodor": 5, "rothschild": 5, "phan": [5, 8], "tiberiu": 5, "nkinyili": 5, "timo": 5, "schick": 5, "timofei": 5, "kornev": 5, "titu": 5, "tunduni": 5, "gerstenberg": 5, "trenton": 5, "trishala": 5, "neeraj": 5, "tushar": 5, "khot": 5, "shultz": 5, "uri": 5, "shaham": 5, "vera": 5, "demberg": 5, "victoria": [5, 8], "nyamai": 5, "vika": 5, "raunak": 5, "vinai": 5, "ramasesh": 5, "udai": 5, "prabhu": 5, "vishakh": 5, "padmakumar": 5, "vivek": [5, 6], "srikumar": [5, 6], "fedu": [5, 8], "wout": 5, "vossen": 5, "xiaoyu": 5, "tong": [5, 8], "xinran": 5, "xinyi": 5, "yadollah": 5, "yaghoobzadeh": 5, "yair": 5, "lakretz": 5, "yangqiu": 5, "yasaman": 5, "bahri": 5, "yichi": 5, "yide": 5, "yifu": 5, "yonatan": 5, "belinkov": 5, "yufang": 5, "seid": 5, "zhuoy": 5, "zijian": 5, "ziji": 5, "zirui": 5, "ziyi": 5, "extrapol": 5, "2206": 5, "04615": 5, "wpn": 5, "yada": 5, "pruksachatkun": 5, "amanpreet": 5, "hill": 5, "stickier": 5, "wsm": 5, "1804": 5, "07461": 5, "wtb": 5, "tai": 5, "borgeaud": 5, "dani": 5, "yogatama": 5, "denni": [5, 8], "donald": 5, "metzler": 5, "ed": 5, "oriol": 5, "vinyal": 5, "dean": 5, "07682": 5, "wdr": 5, "doolei": 5, "manlei": 5, "arka": [5, 8], "pal": 5, "feuer": 5, "siddhartha": 5, "ravid": 5, "shwartz": [5, 8], "ziv": 5, "khalid": [5, 7], "saifullah": 5, "siddartha": 5, "naidu": 5, "chinmai": 5, "hegd": 5, "lecun": 5, "goldstein": 5, "willi": 5, "neiswang": 5, "micah": 5, "goldblum": 5, "19314": 5, "yyh": 5, "baosong": [5, 7], "chengpeng": 5, "chengyuan": [5, 7], "fei": [5, 7], "guant": 5, "haoran": [5, 7], "huan": [5, 7], "jialong": 5, "jialin": 5, "jianhong": [5, 7], "tu": [5, 7], "jianwei": [5, 7], "jianxin": [5, 7], "jin": [5, 6, 8], "jingren": [5, 7], "jinz": 5, "jinzheng": 5, "junyang": [5, 7], "keme": [5, 7], "keqin": [5, 7], "kexin": [5, 7], "mingfeng": [5, 7], "xue": [5, 7, 8], "ni": [5, 6], "pei": [5, 7], "ru": 5, "men": [5, 7], "ruiz": 5, "runji": [5, 7], "shiji": 5, "sinan": 5, "tianhang": 5, "wenbin": 5, "ge": 5, "xiaodong": 5, "deng": 5, "xiaohuan": 5, "xingzhang": [5, 7], "xinyu": [5, 8], "xipin": 5, "xuancheng": [5, 7], "yichang": [5, 7], "wan": [5, 7], "yunfei": 5, "yuqiong": [5, 7], "zhenru": [5, 7], "zhihao": 5, "10671": 5, "zcl24": 5, "zhihan": 5, "cao": 5, "lizi": 5, "openreview": [5, 6], "forum": [5, 6], "aegrf1uy0p": 5, "zc": 5, "siyuan": 5, "zhuang": [5, 8], "zhanghao": 5, "yonghao": 5, "zi": 5, "zhuohan": 5, "xing": [5, 8], "2306": 5, "05685": 5, "huggingface24": 5, "metaai24": 5, "doubl": 6, "steve": [6, 8], "lclm": 6, "simultan": [6, 7, 8], "cutoff": 6, "amayuela": 6, "tail": 6, "kotha": 6, "unifi": [6, 7, 9], "codebas": [6, 7], "ingest": 6, "preprocess": [6, 7, 9], "parser": [6, 9], "microsoft": [6, 7], "autogen": 6, "powerpoint": 6, "ocr": 6, "exif": 6, "metadata": [6, 7], "docker": [6, 7], "container": [6, 7], "xlsx": 6, "text_cont": 6, "ibm": [6, 7, 8], "docx": 6, "pptx": 6, "layout": 6, "llamaindex": 6, "document_convert": 6, "documentconvert": 6, "export_to_markdown": 6, "presenc": 6, "merril": 6, "lynch": 6, "cio": 6, "outlook": 6, "forecast_file_path": 6, "result_md": 6, "forecast_result_docl": 6, "levenshtein": 6, "distanc": 6, "sequencematch": 6, "difflib": 6, "longest": 6, "levenshtein_similar": 6, "text1": 6, "text2": 6, "max_len": 6, "simple_similar": 6, "ratio": [6, 7], "forecast_result_md": 6, "13985705461925346": 6, "17779960707269155": 6, "readabl": 6, "messi": 6, "2025e": 6, "compos": [6, 7, 8], "financial_vari": 6, "financial_forecast": 6, "econforecast": 6, "extract_prompt": 6, "base_prompt": [6, 9], "extract_from_doc": 6, "twice": 6, "md_financi": 6, "docling_financi": 6, "easier": [6, 7, 8, 9], "gdp": 6, "cpi": 6, "fed": 6, "df_md_forecast": 6, "df_docling_forecast": 6, "despit": [6, 7, 9], "underweight": 6, "neutral": [6, 8], "overweight": 6, "chart": 6, "asset_class_docl": 6, "asset_class_md": 6, "df_md": 6, "df_docl": 6, "true_valu": 6, "df_comparison": 6, "cap": 6, "exempt": 6, "markitdown_accuraci": 6, "docling_accuraci": 6, "93": [6, 7, 8], "unstructur": [6, 7, 9], "sector": 6, "convert_and_export_t": 6, "file_path": 6, "doc_convert": 6, "start_tim": [6, 8], "conv_r": 6, "table_df": 6, "export_to_datafram": 6, "end_tim": 6, "2f": 6, "usd": 6, "wtd": 6, "mtd": 6, "ytd": 6, "djia": 6, "926": 6, "amp": 6, "051": 6, "277": 6, "russel": [6, 8], "2000": 6, "msci": 6, "817": [6, 8], "eaf": 6, "319": 6, "107": 6, "01": [6, 7], "66": [6, 8], "92": 6, "municip": 6, "79": [6, 8], "slight": 6, "discretionari": 6, "yellow": 6, "estat": 6, "orang": 6, "stapl": 6, "constructor": 6, "md_llm": 6, "llm_client": 6, "llm_model": 6, "png": 6, "overview": [6, 9], "showcas": 6, "bond": 6, "crude": 6, "oil": 6, "sit": 6, "648": 6, "ounc": 6, "euro": 6, "tactic": 6, "bofa": 6, "circl": [6, 8], "fetch": 6, "reassembl": 6, "max_output_token": 6, "statement": [6, 8], "10k": 6, "diagram": [6, 8], "charactertextsplitt": 6, "tiktoken": [6, 8], "sequenti": 6, "newlin": 6, "cheap": 6, "speciali": 6, "nltk": 6, "spaci": 6, "hierarch": [6, 8], "talk": 6, "theme": [6, 7, 8], "splitter": 6, "surpass": 6, "get_chunk": 6, "chunk_siz": 6, "chunk_overlap": 6, "langchain_text_splitt": 6, "text_splitt": 6, "from_tiktoken_encod": 6, "split_text": 6, "persona": 6, "langchain_cor": [6, 9], "prompttempl": 6, "get_base_prompt_templ": 6, "from_templ": 6, "llmchain": 6, "output_pars": 6, "stroutputpars": 6, "langchain_commun": 6, "chat_model": 6, "chatlitellm": 6, "get_llm_chain": 6, "prompt_templ": [6, 9], "llm_chain": [6, 9], "api_key_label": 6, "upper": 6, "_api_kei": 6, "get_dynamic_prompt_templ": 6, "get_dynamic_prompt_param": 6, "prompt_param": 6, "part_idx": 6, "total_part": 6, "chat_context": 6, "param": 6, "dynamic_prompt_param": 6, "concaten": 6, "generate_report": 6, "input_cont": 6, "llm_model_nam": 6, "report_part": 6, "num_part": 6, "dinam": 6, "priovid": 6, "invok": [6, 9], "cummul": 6, "max_chunk_s": 6, "max_chunk_overlap": 6, "gemini": [6, 7], "apple_report": 6, "report_cont": 6, "report_lin": 6, "splitlin": 6, "total_lin": 6, "quarter_lin": 6, "top_port": 6, "bottom_port": 6, "uncov": [6, 8, 9], "delv": 6, "consol": 6, "fewer": [6, 7, 8], "reaction": 6, "breakdown": [6, 8], "disciplin": 6, "appar": [6, 8], "subhead": 6, "depth": [6, 8], "2m": [6, 7], "harvard": [6, 7], "enrol": 6, "gov": [6, 8], "1039": 6, "birth": [6, 8], "democraci": 6, "tuesdai": 6, "magna": 6, "carta": 6, "trudg": 6, "dens": 6, "conversation": 6, "knowledge_bas": 6, "add_knowledge_bas": 6, "add_cit": 6, "bool": [6, 8], "num_quest": 6, "input_memori": 6, "response_memori": 6, "urls_memori": 6, "extractor": 6, "cic": 6, "citabl": 6, "passag": [6, 8], "corpora": 6, "formatted_cont": 6, "reference_id": 6, "wrapper": [6, 9], "content_gener": 6, "user_instruct": 6, "llmbackend": 6, "cache_ttl": 6, "cachedcont": 6, "display_nam": 6, "due_knowledge_bas": 6, "system_instruct": 6, "compose_prompt": 6, "conversation_config": 6, "ttl": 6, "generativemodel": 6, "from_cached_cont": 6, "cached_cont": 6, "behind": [6, 8], "quiz_inst": 6, "professor": 6, "difficulti": [6, 8], "dataset": [6, 9], "syllabu": 6, "kennedi": 6, "inaugur": 6, "lincoln": 6, "gettysburg": 6, "liberti": 6, "mayflow": 6, "abraham": 6, "gutenberg": 6, "kb": 6, "epub": 6, "pg": 6, "gemini_duo": 6, "genai_duo": 6, "duo": 6, "usage_metadata": 6, "38470": 6, "anytim": 6, "shap": 6, "mckechni": 6, "study_refer": 6, "pg10000": 6, "65363": 6, "pg65363": 6, "quizz": 6, "problemat": [6, 8], "misinterpret": 6, "awp": 6, "alfonso": 6, "liangm": 6, "pan": [6, 8], "wenhu": 6, "lun": 6, "ku": 6, "editor": [6, 8], "acl": [6, 8], "6416": 6, "6432": 6, "bangkok": 6, "thailand": 6, "aclanthologi": [6, 8], "383": 6, "18653": [6, 8], "v1": [6, 7, 8], "ksr24": 6, "suha": 6, "springer": 6, "aditi": 6, "raghunathan": 6, "twelfth": 6, "vrhif2hsrm": 6, "lcd": 6, "jinhyuk": 6, "zhuyun": 6, "dheeru": 6, "dua": 6, "devendra": 6, "sachan": 6, "boratko": 6, "luan": 6, "s\u00e9bastien": 6, "arnold": 6, "vincent": 6, "perot": 6, "siddharth": 6, "dalmia": 6, "hexiang": 6, "panupong": 6, "pasupat": 6, "aida": 6, "amini": 6, "cole": 6, "riedel": 6, "iftekhar": 6, "naim": 6, "ming": [6, 8], "guu": 6, "subsum": 6, "sql": 6, "13121": 6, "lpp": 6, "aleksandra": 6, "piktu": 6, "fabio": [6, 8], "petroni": 6, "vladimir": 6, "karpukhin": 6, "heinrich": 6, "k\u00fcttler": 6, "tau": 6, "yih": 6, "rockt\u00e4schel": 6, "douw": 6, "kiela": 6, "2005": 6, "11401": 6, "nbgc24": 6, "shiyu": 6, "kepe": 6, "bi": 6, "jiafeng": 6, "guo": [6, 8], "xueqi": 6, "cheng": [6, 9], "11375": 6, "11388": 6, "675": 6, "tdw": 6, "jiejun": 6, "zhicheng": 6, "dou": 6, "mang": 6, "weipeng": 6, "ji": 6, "htmlrag": 6, "02959": 6, "zlj": 6, "jiaji": 6, "yun": [6, 9], "metacognit": 6, "1453": 6, "1463": 6, "ny": [6, 8, 9], "usa": [6, 8, 9], "machineri": [6, 9], "1145": [6, 8, 9], "3589334": 6, "3645481": 6, "anthropic4a": 6, "langchain24": 6, "how_to": 6, "merrilllynch24": 6, "weekli": 6, "olui2": 6, "gwmol": 6, "di": 7, "hunter": 7, "photo": 7, "email": 7, "hipaa": 7, "properti": [7, 8], "gdpr": 7, "strict": [7, 8, 9], "iot": 7, "unreli": 7, "impract": 7, "slm": 7, "viabl": 7, "sensor": 7, "interconnect": 7, "frontend": 7, "garner": 7, "traction": 7, "yourself": 7, "aw": [7, 8], "bedrock": 7, "sambanova": 7, "sla": 7, "veloc": 7, "roadmap": 7, "commodit": 7, "winner": 7, "loser": 7, "condens": 7, "clean": 7, "2024t": 7, "versatil": 7, "72b": 7, "med": 7, "bloomberggpt": 7, "underw": 7, "adept": 7, "toxigen": 7, "alnajjar": 7, "13b": [7, 8], "outperform": 7, "32b": 7, "feasibl": 7, "modal": 7, "diagnosi": 7, "patient": 7, "necessit": 7, "deepseek": 7, "flagship": 7, "405b": 7, "pack": 7, "v3": [7, 8], "671": 7, "moe": 7, "mixtur": 7, "3x": [7, 8], "v2": [7, 8], "fraction": 7, "domin": 7, "cautiou": 7, "cautious": 7, "isol": [7, 8], "cpot": 7, "cpit": 7, "tco": 7, "tpot": 7, "ttft": 7, "gpqa": 7, "median": 7, "afford": 7, "lite": 7, "micro": 7, "encod": [7, 8, 9], "cent": 7, "1m": 7, "cheapest": 7, "phi": 7, "half": [7, 8], "permiss": [7, 8], "apach": 7, "simpler": [7, 9], "700m": 7, "100m": 7, "gemma": [7, 9], "grown": 7, "withdraw": 7, "incomplet": [7, 8], "unclear": 7, "15t": 7, "8t": 7, "fineweb": 7, "penedo": 7, "96": [7, 8], "crawl": 7, "snapshot": 7, "ablat": 7, "vital": [7, 8], "favorit": 7, "spawn": 7, "ultrachat": 7, "2024u": 7, "created_job": 7, "fine_tun": 7, "training_fil": 7, "file_id": 7, "ultrachat_chunk_train": 7, "validation_fil": 7, "ultrachat_chunk_ev": 7, "training_step": 7, "0001": 7, "auto_start": 7, "job_id": 7, "toolkit": [7, 8], "sft": 7, "nemo": [7, 8], "codestr": 7, "2024v": 7, "enough": 7, "rewrit": 7, "smolvlm": 7, "mlx": [7, 9], "mlc": 7, "peft": 7, "programm": 7, "graphic": [7, 8], "vram": 7, "mathbf": 7, "x_1": [7, 9], "x_2": [7, 9], "x_n": [7, 9], "x_": [7, 9], "\u03b8": 7, "matrix": [7, 8], "cerebra": 7, "mozilla": 7, "gerganov": 7, "georgi": 7, "overwhelm": [7, 9], "manifesto": 7, "enjoy": 7, "bog": 7, "exploratori": 7, "hacker": 7, "Will": [7, 8], "prototyp": 7, "prematur": 7, "besid": 7, "lighter": 7, "sacrific": 7, "ggml": [7, 9], "disk": 7, "backward": 7, "2024x": 7, "repo": 7, "compil": 7, "linux": 7, "argument": [7, 8, 9], "sudo": 7, "apt": 7, "cmake": 7, "bind": 7, "betlen": 7, "cnv": 7, "llamacpp": 7, "succinct": 7, "ctrl": 7, "interject": 7, "philosoph": 7, "debat": 7, "fulfil": 7, "happi": 7, "responsibli": 7, "bye": 7, "goodby": 7, "port": 7, "127": 7, "curl": [7, 9], "localhost": 7, "bearer": 7, "finish_reason": 7, "deepli": 7, "1734627879": 7, "completion_token": 7, "total_token": 7, "chatcmpl": 7, "5wl2tzjzdmzupvxwp2gcedr8xbpsyhfm": 7, "prompt_n": 7, "prompt_m": 7, "132": 7, "prompt_per_token_m": 7, "prompt_per_second": 7, "77619878666999": 7, "predicted_n": 7, "predicted_m": 7, "1700": 7, "654": [7, 9], "predicted_per_token_m": 7, "36882142857143": 7, "predicted_per_second": 7, "92850867960208": 7, "gbnf": [7, 9], "8pm": 7, "appointmenttim": 7, "appointmentdetail": 7, "handi": 7, "model_path": 7, "llama_cpp": 7, "create_chat_complet": 7, "occupi": 7, "activist": 7, "justin": [7, 8], "tunnei": 7, "ocho": 7, "appach": 7, "cosmopolitan": 7, "libc": 7, "portabl": 7, "durabl": 7, "usabl": [7, 8, 9], "tinyllama": 7, "wget": 7, "jartin": 7, "q5_k_m": 7, "renam": 7, "ex": 7, "chmod": 7, "nobrows": 7, "registri": 7, "nativ": [7, 9], "trai": 7, "familiar": 7, "bare": 7, "ssfl": 7, "sh": [7, 9], "Or": 7, "11434": 7, "chatrespons": 7, "easiest": 7, "rich": [7, 8], "playground": 7, "importantli": [7, 9], "intuit": 7, "beginn": 7, "tensorrt": 7, "trt": 7, "latex": 7, "voic": 7, "pwa": 7, "medium": [7, 8], "gpt4all": 7, "rbac": 7, "q4_k": 7, "q6_k": 7, "mib": 7, "wikitext": 7, "salesforc": 7, "wikipedia": [7, 9], "min_prompt_length": 7, "input_texts_raw": 7, "began": 7, "2010": 7, "valkyria": 7, "chronicl": 7, "forgiv": 7, "newcom": 7, "raita": 7, "honjou": 7, "hitoshi": 7, "sakimoto": 7, "takeshi": 7, "ozawa": 7, "writer": 7, "sung": 7, "escap": 7, "escaped_text": 7, "block_scal": 7, "block": [7, 8], "parenthes": 7, "block_min": 7, "formula": 7, "superblock": 7, "5625": 7, "ieee": 7, "754": 7, "ppl": 7, "exp": 7, "sum_": 7, "log_2": 7, "x_i": [7, 9], "avg": 7, "_i": 7, "corr": 7, "ln": [7, 9], "kullback": 7, "leibler": 7, "entropi": 7, "logit": 7, "d_": 7, "softmax": [7, 9], "sum": 7, "kld": 7, "q2_kresult": 7, "q6": 7, "004": 7, "q2": 7, "112": 7, "q4": 7, "smallest": 7, "390": 7, "67": [7, 8], "81": [7, 8], "462": 7, "614": 7, "170": 7, "q4_k_m": 7, "thread": 7, "16x": 7, "85x": 7, "79x": 7, "ubuntu": 7, "lt": 7, "x86_64": 7, "gnu": 7, "thank": [7, 9], "intel": 7, "i7": 7, "8550u": 7, "15gib": 7, "samsung": 7, "ssd": 7, "970": 7, "evo": 7, "500gb": 7, "1170": 7, "meant": 7, "ai4c": 7, "ai4a": 7, "paperswithcod": [7, 8], "ana24a": 7, "leaderboard": [7, 8], "artificialanalysi": 7, "ana24b": 7, "ana24c": 7, "bc24": 7, "andrei": [7, 8], "abetlen": 7, "dee24": 7, "blob": [7, 9], "deepseek_v3": 7, "fac4": 7, "optimum": 7, "concept_guid": 7, "fac4t": 7, "fac4u": 7, "200k": 7, "ultrachat_200k": 7, "fac4v": 7, "blogpost": 7, "gc24": 7, "ggerganov": [7, 9], "readm": [7, 9], "gc4a": 7, "gc4b": 7, "pka": 7, "guilherm": 7, "hynek": 7, "kydl\u00ed\u010dek": 7, "decant": 7, "finest": 7, "17557": 7, "qwe4b": 7, "qy": 7, "beichen": 7, "tingyu": 7, "zihan": 7, "qiu": 7, "15115": 7, "rev24": 7, "nyt": 7, "harvardlawreview": 7, "timess": 7, "zwa": 7, "wael": 7, "geoffrei": [7, 8], "angu": 7, "arnav": 7, "jefferi": 7, "kinnison": 7, "sherstinski": 7, "piero": 7, "molino": 7, "travi": 7, "addair": 7, "devvret": 7, "310": 7, "2405": 7, "00732": 7, "huggingface4xa": 7, "huggingface4xb": 7, "ibmthink24": 7, "lmstudio24": 7, "lmstudio": 7, "metaai4c": 7, "mozillaocho24": 7, "salesforce24": 7, "immens": 8, "commonplac": 8, "spur": 8, "hartvigsen": 8, "societi": 8, "alarm": 8, "openli": 8, "dolli": 8, "llama2": [8, 9], "emb": 8, "generalist": 8, "injustic": 8, "inequ": 8, "undermin": 8, "perpetu": 8, "displac": 8, "eros": 8, "fake": 8, "deepfak": 8, "distrust": 8, "cyberattack": 8, "spread": 8, "disinform": 8, "inadvert": 8, "interven": 8, "irrevers": 8, "uncheck": 8, "extinct": 8, "race": 8, "incentiv": 8, "shortcut": 8, "stress": 8, "urgent": 8, "reorient": 8, "siam": 8, "edgington": 8, "jailbreak": 8, "promptcraft": 8, "stealth": 8, "sutton": 8, "subtl": 8, "subtleti": 8, "exception": 8, "phrase": 8, "evad": 8, "hqve": 8, "frer": 8, "hplidai": 8, "pl": 8, "hyperion": 8, "coast": 8, "redwood": 8, "tallest": 8, "tree": [8, 9], "routin": 8, "prejudic": 8, "gallego": 8, "leak": 8, "poison": 8, "intention": 8, "inject": 8, "mislead": 8, "exabeam": 8, "finra": 8, "3110": 8, "mandat": 8, "supervisori": 8, "unicef": 8, "empow": 8, "contest": 8, "congress": 8, "enact": 8, "pictur": [8, 9], "sound": 8, "territori": 8, "oversea": 8, "chines": 8, "legitim": 8, "consent": 8, "complaint": 8, "cooper": 8, "extraterritori": 8, "offshor": 8, "draft": 8, "voluntari": 8, "player": 8, "prepared": 8, "compris": 8, "cbrn": 8, "persuas": 8, "autonomi": 8, "gradat": 8, "scorecard": 8, "elig": 8, "advisori": 8, "sag": 8, "shut": 8, "prerequisit": 8, "harden": 8, "asl": 8, "biosafeti": 8, "elev": 8, "warn": [8, 9], "bioweapon": 8, "compartment": 8, "4x": 8, "jump": 8, "paus": 8, "deepmind": 8, "biosecur": 8, "buffer": 8, "formul": [8, 9], "calibr": 8, "promin": 8, "taxonomi": 8, "llamaguard": 8, "20241022": 8, "5x": 8, "alaga": 8, "substandard": 8, "oxford": 8, "wachter": 8, "blur": 8, "ill": 8, "stifl": 8, "suscept": 8, "aadc": 8, "outset": 8, "curricula": 8, "adversari": 8, "thoroughli": 8, "lm": [8, 9], "undergo": 8, "280b": 8, "cai": [8, 9], "enshrin": 8, "evas": 8, "resort": 8, "avenu": 8, "cambria": 8, "inherit": 8, "influenti": 8, "debias": 8, "plausibl": 8, "occurr": 8, "phish": 8, "clarifi": 8, "toler": 8, "checklist": 8, "abus": 8, "ux": 8, "architect": 8, "retrofit": 8, "promptli": 8, "dashboard": 8, "misalign": 8, "star": 8, "postpon": 8, "combat": 8, "counter": 8, "traffic": 8, "frustrat": 8, "workaround": 8, "silo": 8, "hierarchi": 8, "mcq": 8, "regex": [8, 9], "joint": 8, "facet": 8, "purpl": 8, "opensafetylab": 8, "salad_bench_dataset": 8, "base_set": 8, "gptfuzzer": 8, "auto": [8, 9], "qid": 8, "o1": 8, "supremaci": 8, "o53": 8, "o14": 8, "o5": 8, "o65": 8, "plagiar": 8, "o16": 8, "o6": 8, "o47": 8, "campaign": 8, "o12": 8, "o52": 8, "surveil": 8, "spous": 8, "know": 8, "o13": 8, "ncount": 8, "21318": 8, "8756": 8, "6486": 8, "o2": 8, "1717": 8, "o4": 8, "1477": 8, "o3": 8, "socioeconom": 8, "851": 8, "int64": 8, "gen": 8, "15433": 8, "hh": 8, "4184": 8, "659": 8, "advbench": 8, "230": 8, "189": 8, "toxicchat": 8, "anyth": 8, "misconcept": 8, "ingrain": 8, "mc1": 8, "singular": 8, "choices4": 8, "mc2": 8, "set4": 8, "scorer": 8, "correctli": [8, 9], "truthful_qa": 8, "truthfulqa_dataset": 8, "multiple_choic": 8, "best_answ": 8, "correct_answ": 8, "incorrect_answ": 8, "watermelon": 8, "digest": 8, "noth": 8, "stomach": 8, "sick": 8, "wonderopoli": 8, "wonder": 8, "belli": 8, "swallow": 8, "dream": 8, "die": 8, "indigest": 8, "unconsci": 8, "excret": 8, "asr": 8, "r2d2": 8, "wider": [8, 9], "mass": 8, "destruct": 8, "asynchron": 8, "webpurifi": 8, "protectai": 8, "comprehend": 8, "amazon": 8, "nvidia": [8, 9], "keyword": 8, "toolset": 8, "nemmo": 8, "synchron": 8, "nemoguardrail": 8, "llmrail": 8, "railsconfig": 8, "from_path": 8, "rail": 8, "hello": 8, "ministr": 8, "mistralai": 8, "mistral_api_kei": 8, "moderate_chat": 8, "omni": 8, "pprint": 8, "to_json": 8, "threaten": 8, "illicit": 8, "granit": 8, "guardian": 8, "consortium": 8, "11b": 8, "begin_of_text": 8, "start_header_id": 8, "end_header_id": 8, "unsafe_categori": 8, "user_message_1": 8, "model_answer_1": 8, "comma": 8, "eot_id": 8, "eom_id": 8, "denot": 8, "s1": 8, "s2": 8, "s3": 8, "s4": 8, "s5": 8, "defam": 8, "s6": 8, "s7": 8, "s8": 8, "s9": 8, "s10": 8, "s11": 8, "s12": 8, "s13": 8, "padhi": 8, "atla": 8, "2b": 8, "hap": 8, "38m": 8, "125m": 8, "shieldgemma": 8, "judge_prompt": 8, "american": 8, "vandal": 8, "underag": 8, "drink": 8, "vulgar": 8, "obscen": 8, "racism": 8, "derogatori": 8, "firearm": 8, "safety_scor": 8, "IN": 8, "borderlin": 8, "verdict": 8, "boolean": [8, 9], "brief": 8, "rational": 8, "paramount": [8, 9], "evenli": 8, "good_sampl": 8, "bad_sampl": 8, "2024z": 8, "surg": 8, "scam": 8, "get_profanity_sampl": 8, "show_stat": 8, "current_dir": 8, "getcwd": 8, "data_path": 8, "profanity_en": 8, "random_st": 8, "ncategori": 8, "category_count": 8, "category_1": 8, "1f": 8, "profanity_sampl": 8, "nsampl": 8, "anatomi": 8, "slur": 8, "182": 8, "bodili": 8, "fluid": 8, "excrement": 8, "insult": 8, "mental": 8, "disabl": 8, "jap": 8, "babi": [8, 9], "batter": 8, "crazi": 8, "sob": 8, "fukka": 8, "sh1t3": 8, "get_salad_sampl": 8, "salad_sampl": 8, "tortur": 8, "porn": 8, "sin": 8, "sight": 8, "god": 8, "embezzl": 8, "xanax": 8, "alcohol": 8, "get_good_sampl": 8, "min_scor": 8, "reichstag": 8, "profanity_data": 8, "salad_data": 8, "good_data": 8, "all_data": 8, "prompt_sampl": 8, "is_unsaf": 8, "counti": 8, "holli": 8, "ridg": 8, "nc": 8, "town": 8, "onslow": 8, "carolina": 8, "diver": 8, "underwat": 8, "maze": 8, "coral": 8, "treasur": 8, "vivid": 8, "sensori": 8, "emot": 8, "labyrinthin": 8, "reef": 8, "suspens": 8, "obstacl": 8, "creatur": 8, "nomin": 8, "nobel": 8, "literatur": 8, "love": 8, "logo": 8, "thief": 8, "rob": 8, "famou": 8, "nstatist": 8, "source_stat": 8, "type_stat": 8, "plug": 8, "safetyvalid": 8, "validationresult": 8, "dataclass": 8, "abstractmethod": 8, "llmguardvalid": 8, "scanner": 8, "bantop": 8, "llm_guard": 8, "input_scann": 8, "scan_prompt": 8, "matchtyp": 8, "default_banned_top": 8, "banned_top": 8, "super": 8, "banned_topics_scann": 8, "use_onnx": 8, "toxicity_scann": 8, "match_typ": 8, "fail_fast": 8, "unsafe_scann": 8, "gun": 8, "cool": 8, "hunt": 8, "deer": 8, "dad": 8, "mistralvalid": 8, "hate_and_discrimin": 8, "violence_and_threat": 8, "dangerous_and_criminal_cont": 8, "selfharm": 8, "openaivalid": 8, "attr": 8, "dir": 8, "getattr": 8, "illicit_viol": 8, "llmjudgevalid": 8, "prompt_path": 8, "llmjudg": 8, "filenotfounderror": 8, "slice": 8, "elaps": 8, "score_valid": 8, "id_": 8, "validator_inst": 8, "validation_result": 8, "elapsed_tim": 8, "prompt_sample_id": 8, "validator_nam": 8, "scoring_prompt": 8, "scoring_result": 8, "01536": 8, "34098": 8, "497136": 8, "546416": 8, "calculate_validator_metr": 8, "bad_sourc": 8, "good_sourc": 8, "tpr": 8, "fpr": 8, "f1_score": 8, "tn": 8, "fp": 8, "fn": 8, "tp": 8, "255": 8, "74": 8, "238": 8, "130": 8, "256": 8, "149": 8, "213": 8, "190": 8, "235": 8, "63": 8, "222": 8, "480": 8, "157": 8, "487": 8, "495": 8, "482": 8, "667": 8, "248": 8, "466": 8, "143": 8, "355": 8, "slowest": 8, "room": 8, "false_posit": 8, "left_on": 8, "right_on": 8, "persian": 8, "teenag": 8, "\u0686\u0627\u0644\u0634": 8, "\u0645\u0648\u0645\u0648": 8, "\u06cc\u06a9": 8, "\u062d\u0642\u0647": 8, "\u0648": 8, "\u0627\u0641\u0633\u0627\u0646\u0647": 8, "\u0645\u062d\u0644\u06cc": 8, "\u0627\u0633\u062a": 8, "\u06a9\u0647": 8, "\u0634\u0627\u06cc\u0639\u0647": 8, "\u0622\u0646": 8, "\u062f\u0631": 8, "\u0634\u0628\u06a9\u0647": 8, "\u0647\u0627\u06cc": 8, "\u0627\u062c\u062a\u0645\u0627\u0639\u06cc": 8, "\u067e\u062e\u0634": 8, "\u0634\u062f": 8, "\u06af\u0632\u0627\u0631\u0634": 8, "\u06a9\u0627\u0631\u0628\u0631\u06cc": 8, "\u0628\u0627": 8, "\u0646\u0627\u0645": 8, "\u06a9\u0648\u062f\u06a9\u0627\u0646": 8, "\u0646\u0648\u062c\u0648\u0627\u0646\u0627\u0646": 8, "\u0631\u0627": 8, "\u0628\u0647": 8, "\u0627\u0639\u0645\u0627\u0644": 8, "\u062e\u0637\u0631\u0646\u0627\u06a9\u06cc": 8, "\u0645\u0627\u0646\u0646\u062f": 8, "\u062e\u0648\u062f\u0632\u0646\u06cc": 8, "\u062e\u0648\u062f\u06a9\u0634\u06cc": 8, "\u062a\u0631\u063a\u06cc\u0628": 8, "\u0645\u06cc": 8, "\u06a9\u0646\u062f": 8, "\u0648\u062c\u0648\u062f": 8, "\u0635\u062d\u0628\u062a": 8, "\u0632\u06cc\u0627\u062f\u06cc": 8, "\u0645\u0648\u0631\u062f": 8, "\u0627\u06cc\u0646": 8, "\u062a\u0639\u062f\u0627\u062f": 8, "\u0634\u06a9\u0627\u06cc\u0627\u062a": 8, "\u0648\u0627\u0642\u0639\u06cc": 8, "\u0628\u0633\u06cc\u0627\u0631": 8, "\u06a9\u0645": 8, "\u0628\u0648\u062f": 8, "\u067e\u0644\u06cc\u0633": 8, "\u0635\u062f\u0645\u0647": 8, "\u062f\u06cc\u062f\u0646": 8, "\u062f\u0644\u06cc\u0644": 8, "\u062a\u0623\u062b\u06cc\u0631": 8, "\u0645\u0633\u062a\u0642\u06cc\u0645": 8, "\u067e\u062f\u06cc\u062f\u0647": 8, "\u062a\u0623\u06cc\u06cc\u062f": 8, "\u0646\u06a9\u0631\u062f\u0647": 8, "\u062a\u0631\u0633": 8, "\u0646\u06af\u0631\u0627\u0646\u06cc": 8, "\u0627\u06cc\u062c\u0627\u062f": 8, "\u0634\u062f\u0647": 8, "\u0628\u06cc\u0634\u062a\u0631": 8, "\u0627\u0632": 8, "\u062e\u0648\u062f": 8, "\u0631\u0633\u0627\u0646\u0647": 8, "\u0647\u0627": 8, "\u0637\u0648\u0631\u06cc": 8, "\u062e\u06cc\u0631\u06cc\u0647": 8, "\u0647\u0634\u062f\u0627\u0631": 8, "\u062f\u0627\u062f\u0646\u062f": 8, "\u0622\u0633\u06cc\u0628": 8, "\u0627\u0646\u062a\u0638\u0627\u0631\u0627\u062a": 8, "\u0645\u062d\u062a\u0648\u0627\u06cc": 8, "\u062e\u0634\u0648\u0646\u062a": 8, "\u0622\u0645\u06cc\u0632": 8, "\u0627\u06cc\u0646\u062a\u0631\u0646\u062a": 8, "\u06af\u0641\u062a\u0647": 8, "\u0634\u0648\u062f": 8, "\u0627\u0648\u0644\u06cc\u0646": 8, "\u0628\u0627\u0631": 8, "\u0633\u0627\u0644": 8, "\u06f2\u06f0\u06f1\u06f8": 8, "\u067e\u0633": 8, "\u0622\u0646\u06a9\u0647": 8, "\u0631\u0648\u0632\u0646\u0627\u0645\u0647": 8, "\u0627\u0646\u062f\u0648\u0646\u0632\u06cc\u0627\u06cc\u06cc": 8, "\u062e\u0628\u0631": 8, "\u062f\u062e\u062a\u0631": 8, "\u06f1\u06f2": 8, "\u0633\u0627\u0644\u0647": 8, "\u062f\u0627\u062f": 8, "\u0645\u0648\u0636\u0648\u0639": 8, "\u062c\u0647\u0627\u0646\u06cc": 8, "\u062a\u0628\u062f\u06cc\u0644": 8, "\u0645\u062c\u0633\u0645\u0647": 8, "\u0647\u0646\u0631\u0645\u0646\u062f": 8, "\u0698\u0627\u067e\u0646\u06cc": 8, "\u0647\u0631": 8, "\u0686\u0646\u062f": 8, "\u0634\u0627\u06cc\u062f": 8, "\u0646\u06af\u0627\u0647": 8, "\u0628\u0639\u0636\u06cc": 8, "\u0632\u06cc\u0628\u0627": 8, "\u0646\u0628\u0627\u0634\u062f": 8, "\u0627\u0645\u0627": 8, "\u06a9\u0627\u0645\u0644\u0627": 8, "\u0628\u06cc": 8, "\u062e\u0637\u0631": 8, "\u0627\u06cc\u0631\u0627\u0646": 8, "\u0645\u062f\u062a": 8, "\u0628\u06cc\u0646": 8, "\u06a9\u0627\u0631\u0628\u0631\u0627\u0646": 8, "\u0645\u0637\u0631\u062d": 8, "\u0633\u0627\u0644\u06cc": 8, "\u0633\u0631\u0627\u0633\u0631": 8, "\u062c\u0647\u0627\u0646": 8, "\u0645\u0634\u0627\u0628\u0647\u06cc": 8, "\u0628\u0631\u0627\u06cc": 8, "\u0648\u0627\u0644\u062f\u06cc\u0646": 8, "\u06a9\u0631\u062f\u0647": 8, "\u0627\u0641\u0631\u0627\u062f": 8, "\u0686\u0647": 8, "\u06a9\u0627\u0631\u06cc": 8, "\u062f\u0639\u0648\u062a": 8, "tourist": 8, "distress": 8, "polish": 8, "galician": 8, "dzisiaj": 8, "szwecji": 8, "innych": 8, "bogatych": 8, "krajach": 8, "ludzi": 8, "u\u017cywaj\u0105": 8, "mn\u00f3stwo": 8, "najr\u00f3\u017cniejszych": 8, "urz\u0105dze\u0144": 8, "hox": 8, "suecia": 8, "outro": 8, "pa\u00eds": 8, "rico": 8, "xent": 8, "moita": 8, "m\u00e1quina": 8, "diferent": 8, "\u0142\u00f3dka": 8, "zaczyna": 8, "ton\u0105\u0107": 8, "tury\u015bci": 8, "wracaj\u0105": 8, "statek": 8, "dom\u00f3w": 8, "gdzie": 8, "opowiadaj\u0105": 8, "tym": 8, "jak": 8, "zostali": 8, "zaatakowani": 8, "surprisingli": 8, "shelf": 8, "unsettl": 8, "paradox": 8, "harbor": 8, "wisdom": 8, "aspir": 8, "technologist": 8, "disciplinari": 8, "ethicist": 8, "policymak": 8, "asa24": 8, "jide": 8, "jona": 8, "schuett": 8, "marku": 8, "anderljung": 8, "08751": 8, "bhy": 8, "hinton": 8, "pieter": 8, "abbeel": 8, "trevor": 8, "darrel": 8, "yuval": 8, "harari": 8, "ya": 8, "lan": 8, "shai": 8, "shalev": 8, "gillian": 8, "hadfield": 8, "clune": 8, "tegan": 8, "maharaj": 8, "hutter": 8, "at\u0131l\u0131m": 8, "g\u00fcne\u015f": 8, "baydin": 8, "sheila": 8, "mcilraith": 8, "qiqi": 8, "ashwin": 8, "acharya": 8, "anca": 8, "dragan": 8, "philip": 8, "torr": 8, "kahneman": 8, "s\u00f6ren": 8, "mindermann": 8, "amid": 8, "384": 8, "6698": 8, "1126": 8, "adn0117": 8, "bbc": 8, "emili": 8, "braca": 8, "israel": 8, "carter": 8, "hafsa": 8, "kanchwala": 8, "khojasteh": 8, "charli": 8, "landow": 8, "luo": 8, "magarelli": 8, "mirin": 8, "averi": 8, "moyer": 8, "kayla": 8, "simpson": 8, "amelia": 8, "skawinski": 8, "heverin": 8, "23308": 8, "bmc": 8, "dillon": 8, "brendan": 8, "murphi": 8, "khachaturov": 8, "gleav": 8, "kellin": 8, "pelrin": 8, "2408": [8, 9], "02946": 8, "cmm": 8, "erik": 8, "lorenzo": 8, "malandri": 8, "mercorio": 8, "navid": 8, "nobani": 8, "seveso": 8, "15248": 8, "edg24": 8, "exa24": 8, "cyber": 8, "grb": 8, "rossi": 8, "barrow": 8, "mehrab": 8, "tanjim": 8, "sungchul": 8, "franck": 8, "dernoncourt": 8, "ruiyi": 8, "nesreen": 8, "2309": 8, "00770": 8, "h44z": 8, "hgp": 8, "saadia": 8, "hamid": 8, "palangi": 8, "dipankar": 8, "ec": 8, "kamar": 8, "oxi": 8, "smaranda": 8, "muresan": 8, "preslav": 8, "nakov": 8, "alin": 8, "villavicencio": 8, "60th": 8, "3309": 8, "3326": 8, "dublin": 8, "hym": 8, "weijiang": 8, "weitao": 8, "weihong": 8, "zhangyin": 8, "haotian": 8, "qianglong": 8, "weihua": 8, "xiaocheng": 8, "bing": 8, "dx": 8, "3703155": 8, "ldw": 8, "lijun": 8, "ruohui": 8, "xuhao": 8, "wangmeng": 8, "zuo": 8, "dahua": 8, "qiao": 8, "shao": 8, "05044": 8, "mpy": 8, "xuwang": 8, "zifan": 8, "norman": 8, "mu": 8, "elham": 8, "sakhae": 8, "nathaniel": 8, "forsyth": 8, "04249": 8, "ma24": 8, "mlc24": 8, "illumin": 8, "ailumin": 8, "oaa": 8, "adler": 8, "ahmad": 8, "ilg": 8, "akkaya": 8, "florencia": 8, "leoni": 8, "aleman": 8, "janko": 8, "altenschmidt": 8, "altman": 8, "shyamal": 8, "anadkat": 8, "avila": 8, "valeri": 8, "balcom": 8, "baltescu": 8, "haim": 8, "belgum": 8, "irwan": 8, "bello": 8, "jake": 8, "berdin": 8, "bernadett": 8, "shapiro": 8, "berner": 8, "lenni": 8, "bogdonoff": 8, "boiko": 8, "madelain": 8, "boyd": 8, "luisa": 8, "brakman": 8, "button": 8, "rosi": 8, "campbel": 8, "cann": 8, "brittani": 8, "carei": 8, "carlson": 8, "rori": 8, "carmichael": 8, "che": 8, "foti": 8, "sulli": 8, "rubi": 8, "chess": 8, "chester": 8, "cho": 8, "hyung": 8, "won": 8, "chung": 8, "jeremiah": 8, "currier": 8, "yunx": 8, "cori": 8, "decareaux": 8, "degri": 8, "deutsch": 8, "devil": 8, "dhar": 8, "dowl": 8, "dun": 8, "adrien": 8, "ecoffet": 8, "atti": 8, "eleti": 8, "tyna": 8, "elound": 8, "farhi": 8, "niko": 8, "sim\u00f3n": 8, "posada": 8, "fishman": 8, "juston": 8, "isabella": 8, "fulford": 8, "georg": 8, "gibson": 8, "vik": 8, "tarun": 8, "gogineni": 8, "goh": 8, "rapha": 8, "gontijo": 8, "lope": 8, "gordon": 8, "morgan": 8, "grafstein": 8, "yufei": 8, "hallaci": 8, "heaton": 8, "johann": 8, "heideck": 8, "hickei": 8, "wade": 8, "hoeschel": 8, "houghton": 8, "kenni": 8, "hsu": 8, "shengli": 8, "joost": 8, "huizinga": 8, "shawn": 8, "joann": 8, "jang": 8, "roger": 8, "haozhun": 8, "shino": 8, "jomoto": 8, "billi": 8, "jonn": 8, "tomer": 8, "kaftan": 8, "\u0142ukasz": 8, "kamali": 8, "ingmar": 8, "kanitscheid": 8, "tabarak": 8, "khan": 8, "logan": 8, "kilpatrick": 8, "jong": 8, "wook": 8, "christina": 8, "yongjik": 8, "hendrik": 8, "kirchner": 8, "kiro": 8, "matt": 8, "kokotajlo": 8, "kondraciuk": 8, "kondrich": 8, "konstantinidi": 8, "kosic": 8, "vishal": 8, "kuo": 8, "lamp": 8, "ikai": 8, "teddi": 8, "jade": 8, "leung": 8, "chak": 8, "lim": 8, "molli": 8, "mateusz": 8, "litwin": 8, "theresa": 8, "lopez": 8, "patricia": 8, "lue": 8, "makanju": 8, "malfacini": 8, "markov": 8, "yaniv": 8, "markovski": 8, "bianca": 8, "mayn": 8, "mckinnei": 8, "christin": 8, "mcleavei": 8, "mcmillan": 8, "mcneil": 8, "aalok": 8, "menick": 8, "mishchenko": 8, "vinni": 8, "monaco": 8, "murk": 8, "m\u00e9ly": 8, "ashvin": 8, "nair": 8, "reiichiro": 8, "nakano": 8, "rajeev": 8, "nayak": 8, "arvind": 8, "neelakantan": 8, "hyeonwoo": 8, "noh": 8, "keef": 8, "jakub": 8, "pachocki": 8, "palermo": 8, "ashlei": 8, "pantuliano": 8, "parish": 8, "emi": 8, "parparita": 8, "passo": 8, "perelman": 8, "belbut": 8, "pere": 8, "pokorni": 8, "pokrass": 8, "vitchyr": 8, "pong": 8, "tolli": 8, "powel": 8, "bori": 8, "proehl": 8, "rae": 8, "ramesh": 8, "franci": 8, "kendra": 8, "rimbach": 8, "carl": 8, "rotst": 8, "roussez": 8, "saltarelli": 8, "ted": 8, "sander": 8, "schnurr": 8, "selsam": 8, "kyla": 8, "sheppard": 8, "toki": 8, "sherbakov": 8, "shieh": 8, "shoker": 8, "pranav": 8, "szymon": 8, "sidor": 8, "sigler": 8, "sitkin": 8, "sokolowski": 8, "natali": 8, "staudach": 8, "madelein": 8, "phil": 8, "tootoonchian": 8, "tseng": 8, "preston": 8, "tuggl": 8, "turlei": 8, "juan": 8, "cer\u00f3n": 8, "urib": 8, "vallon": 8, "vijayvergiya": 8, "jai": 8, "alvin": 8, "ward": 8, "cj": 8, "weinmann": 8, "akila": 8, "welihinda": 8, "jiayi": 8, "weng": 8, "lilian": 8, "wiethoff": 8, "willner": 8, "wolrich": 8, "lauren": 8, "workman": 8, "sherwin": 8, "yoo": 8, "zeller": 8, "shengjia": 8, "juntang": 8, "zhuk": 8, "2303": 8, "08774": 8, "pnc": 8, "inkit": 8, "manish": 8, "nagireddi": 8, "giandomenico": 8, "cornacchia": 8, "subhajit": 8, "chaudhuri": 8, "tejaswini": 8, "pedapati": 8, "pierr": 8, "dognin": 8, "keerthiram": 8, "murugesan": 8, "miehl": 8, "santill\u00e1n": 8, "kieran": 8, "giulio": 8, "zizzo": 8, "muhammad": 8, "zaid": 8, "hame": 8, "purcel": 8, "desmond": 8, "ing": 8, "vejsbjerg": 8, "dali": 8, "hind": 8, "werner": 8, "geyer": 8, "ambrish": 8, "rawat": 8, "kush": 8, "varshnei": 8, "prasanna": 8, "sattigeri": 8, "07724": 8, "saffron": 8, "ring": 8, "aslanid": 8, "glaes": 8, "nat": 8, "mcalees": 8, "irv": 8, "2202": 8, "03286": 8, "szw": 8, "qinghua": 8, "higham": 8, "gorban": 8, "bastouni": 8, "ivan": 8, "tyukin": 8, "12670": 8, "vsk": 8, "simplesafetytest": 8, "2311": 8, "08370": 8, "wmr24": 8, "sandra": 8, "brent": 8, "mittelstadt": 8, "duti": 8, "royal": 8, "240197": 8, "royalsocietypublish": 8, "1098": 8, "rso": 8, "ylx24": 8, "jiahao": 8, "xingwei": 8, "zyi": 8, "shune": 8, "lyumanshan": 8, "jingyu": 8, "shui": 8, "haobin": 8, "pengfei": 8, "hewu": 8, "ghost": 8, "14931": 8, "zho24": 8, "amazonwservices24": 8, "anthropic24": 8, "cdn": 8, "1adf000c8f675958c2ee23805d91aaade1cd4613": 8, "centerfasafety24a": 8, "centerforaisafeti": 8, "centerfasafety24b": 8, "deepmind24": 8, "googleapi": 8, "fsf": 8, "europeanmagency24": 8, "ema": 8, "europa": 8, "activities_en": 8, "financialirauthority24": 8, "ibm24": 8, "watsonx": 8, "saa": 8, "libraryocongress23": 8, "loc": 8, "mistralai24": 8, "mlsteam24": 8, "mlsafeti": 8, "nationaliosatechnology24": 8, "nist": 8, "itl": 8, "nvidia24": 8, "openai24a": 8, "openai24b": 8, "opensafetylab24a": 8, "opensafetylab24b": 8, "protectai24": 8, "surgeai24": 8, "ukgovernment24": 8, "unicef24": 8, "innocenti": 8, "julia": 9, "easili": 9, "trial": 9, "wrangl": 9, "hoc": 9, "unwant": 9, "overflow": 9, "twitter": 9, "youtub": 9, "ldot": 9, "prod_": 9, "syntact": 9, "xml": 9, "invalid": 9, "delic": 9, "heart": 9, "ttt": 9, "itt": 9, "po": 9, "nousresearch": 9, "herm": 9, "person1": 9, "q1": 9, "person2": 9, "json_format": 9, "response_cont": 9, "is_json": 9, "myjson": 9, "nest": 9, "conceptu": 9, "unend": 9, "whitespac": 9, "throw": 9, "somewher": 9, "json_object": 9, "circul": 9, "vertex": 9, "went": 9, "secextract": 9, "mentioned_ent": 9, "mentioned_plac": 9, "extract_from_sec_fil": 9, "sec_filing_text": 9, "hint": 9, "prompt_extract": 9, "sec_extract": 9, "washington": 9, "beg": 9, "unnorm": 9, "0325": 9, "strongest": 9, "greedi": 9, "bfloat16": 9, "device_map": 9, "src": 9, "python3": 9, "nvml": 9, "return_tensor": 9, "pt": 9, "inference_mod": 9, "last_token_logit": 9, "next_token_prob": 9, "nn": 9, "dim": 9, "top_k_prob": 9, "top_k_indic": 9, "topk": 9, "top_k_token": 9, "decod": 9, "idx": 9, "skip_special_token": 9, "prob": 9, "0305": 9, "0197": 9, "0106": 9, "0093": 9, "logitsprocessor": 9, "logits_processor": 9, "logitsprocessorlist": 9, "customlogitsprocessor": 9, "intermediari": 9, "input_id": 9, "__call__": 9, "longtensor": 9, "batch_siz": 9, "sequence_length": 9, "floattensor": 9, "vocab_s": 9, "mask": 9, "pick": 9, "yesnologitsprocessor": 9, "initial_length": 9, "fill_": 9, "inf": 9, "debug": 9, "yes_token": 9, "add_special_token": 9, "no_token": 9, "yes_no_logit": 9, "yes_no_prob": 9, "yes_prob": 9, "no_prob": 9, "yes_mask": 9, "1e4": 9, "NO": 9, "generation_output_control": 9, "uncontrol": 9, "generation_output": 9, "4263": 9, "5737": 9, "10407": 9, "4607": 9, "6250": 9, "9219": 9, "helper": 9, "model_output": 9, "gen_output": 9, "batch_decod": 9, "clean_up_tokenization_spac": 9, "classic": 9, "italian": 9, "willard": 9, "louf": 9, "reformul": 9, "finit": 9, "fsm": 9, "s_": 9, "s_t": 9, "s_1": 9, "tild": 9, "odot": 9, "rightarrow": 9, "wise": 9, "thien": 9, "automaton": 9, "dfa": 9, "outgo": 9, "renorm": 9, "yy": 9, "ever": 9, "aa": 9, "lwai": 9, "prop": 9, "yynnaa": 9, "malform": 9, "sec_extraction_outlin": 9, "zsp": 9, "zicorp": 9, "with_structured_output": 9, "runnabl": 9, "typeddict": 9, "qu": 9, "langchain_openai": 9, "chatopenai": 9, "chatprompttempl": 9, "extract_from_sec_filing_langchain": 9, "structured_llm": 9, "from_messag": 9, "sec_extraction_langchain": 9, "bnf": 9, "backu": 9, "naur": 9, "fssl": 9, "extract_entities_from_sec_fil": 9, "ollama_structured_output_prompt_suffix": 9, "ollama_structured_output_temperatur": 9, "uncensor": 9, "model_json_schema": 9, "response_json": 9, "sharpli": 9, "exllama2": 9, "zoo": 9, "furthermor": 9, "nonetheless": 9, "extran": 9, "dispar": 9, "preval": 9, "peer": 9, "speak": 9, "aider": 9, "outweigh": 9, "rebutt": 9, "dottxt": 9, "reproduct": 9, "paint": 9, "flaw": 9, "uneven": 9, "conflat": 9, "drawback": 9, "pfiffer": 9, "wrestl": 9, "aid24": 9, "dot24": 9, "demo": 9, "gge24": 9, "lan4b": 9, "lww": 9, "xun": 9, "hanyu": 9, "yezhaohui": 9, "shichao": 9, "simin": 9, "shunyu": 9, "feiyu": 9, "xiong": 9, "12599": 9, "llf": 9, "xieyang": 9, "frederick": 9, "fiannaca": 9, "terri": 9, "koo": 9, "dixon": 9, "ea": 9, "3613905": 9, "3650756": 9, "xuan": 9, "hai": 9, "nguyen": 9, "ngoc": 9, "tiviati": 9, "hieu": 9, "dao": 9, "shafiq": 9, "joti": 9, "kenji": 9, "kawaguchi": 9, "nanci": 9, "min": 9, "kan": 9, "08656": 9, "nou24": 9, "out24": 9, "twt": 9, "zhi": 9, "kuang": 9, "tsai": 9, "chieh": 9, "hung": 9, "nung": 9, "02442": 9, "tt24": 9, "vivien": 9, "vivien000": 9, "wl23": 9, "r\u00e9mi": 9, "09702": 9, "guidanceai24": 9, "nvidia4a": 9, "wikipediacontributors24": 9, "wiktionari": 9, "naur_form": 9}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"about": [0, 2], "book": [0, 2], "content": [0, 3, 4, 5, 6, 7, 8, 9], "core": 0, "challeng": 0, "we": 0, "ll": 0, "address": 0, "A": [0, 2, 3, 4], "practic": [0, 2, 7, 9], "approach": [0, 4, 8], "an": 0, "open": [0, 2, 7], "sourc": [0, 2, 7], "note": [0, 3], "perspect": 0, "who": 0, "thi": 0, "i": [0, 3, 6], "For": 0, "outcom": 0, "prerequisit": 0, "set": 0, "up": 0, "your": [0, 7], "environ": 0, "code": 0, "repositori": 0, "python": 0, "setup": [0, 3], "api": [0, 8], "kei": [0, 5], "configur": 0, "troubleshoot": 0, "common": [0, 8], "issu": 0, "author": 0, "prefac": [1, 2], "tame": 2, "llm": [2, 4, 5, 7, 8], "guid": 2, "pitfal": [2, 8], "softwar": [2, 5], "chapter": 2, "1": [2, 8], "The": [2, 4, 5, 7], "eval": [2, 5, 8], "gap": [2, 5], "2": [2, 7, 8], "manag": [2, 6], "input": [2, 6], "data": [2, 3, 6], "3": [2, 8], "structur": [2, 6, 9], "output": [2, 9], "4": [2, 8], "safeti": [2, 8], "5": [2, 8], "prefer": [2, 3], "base": [2, 3, 5, 6, 8], "align": [2, 3], "6": [2, 8], "local": [2, 7], "7": 2, "fall": [2, 4], "cost": [2, 4, 7], "paradox": [2, 4], "8": 2, "frontier": 2, "appendix": 2, "tool": [2, 5, 7, 8, 9], "resourc": 2, "introduct": [3, 5, 6, 7, 8, 9], "from": 3, "raw": 3, "capabl": 3, "On": 3, "misalign": 3, "languag": 3, "model": [3, 5, 7], "human": 3, "supervis": 3, "fine": [3, 7, 9], "tune": [3, 7, 9], "sft": 3, "augment": [3, 6], "post": [3, 9], "train": 3, "answer": 3, "limit": 3, "collaps": 3, "fake": 3, "case": [3, 6, 7, 8], "studi": [3, 6, 7, 8], "polici": [3, 8], "experiment": 3, "deliver": 3, "smollm2": 3, "dataset": [3, 5, 7, 8], "synthet": 3, "gener": [3, 5, 6, 8], "user": [3, 8], "prompt": [3, 7, 9], "reject": 3, "respons": 3, "chosen": 3, "dpo": 3, "optim": [3, 4], "prepar": 3, "vibe": 3, "check": [3, 4], "evalu": [3, 5, 8], "discuss": [3, 6, 9], "conclus": [3, 4, 5, 6, 7, 8, 9], "refer": [3, 4, 5, 6, 7, 8, 9], "why": 4, "matter": 4, "more": 4, "than": 4, "ever": 4, "right": 4, "size": 4, "strateg": 4, "metric": [4, 5], "requir": [4, 5], "busi": 4, "perform": [4, 7], "oper": 4, "technic": [4, 8], "quantiz": [4, 7], "list": 4, "non": 5, "determinist": 5, "machin": 5, "emerg": 5, "properti": 5, "problem": [5, 9], "statement": [5, 9], "tradit": 5, "v": [5, 7], "design": [5, 8], "applic": 5, "test": 5, "matrix": 5, "conceptu": 5, "overview": 5, "consider": 5, "task": [5, 7], "benchmark": [5, 7, 8], "leaderboard": 5, "lightev": 5, "mmlu": 5, "econometr": 5, "sampl": [5, 8], "famili": [5, 7], "us": [5, 6], "langsmith": 5, "promptfoo": 5, "comparison": [5, 7, 9], "pars": 6, "document": 6, "markitdown": 6, "docl": 6, "framework": [6, 8, 9], "extract": 6, "retriev": 6, "chunk": 6, "contextu": 6, "link": 6, "long": 6, "form": 6, "ii": 6, "github": 6, "rag": 6, "iii": 6, "quiz": 6, "citat": 6, "implement": [6, 8], "exampl": 6, "usag": 6, "choos": 7, "suitabl": 7, "result": 7, "llama": 7, "licens": 7, "commun": 7, "support": 7, "custom": [7, 8], "mistral": [7, 8], "decemb": 7, "22": 7, "2024": 7, "deploy": 7, "serv": 7, "cpp": 7, "llamafil": 7, "ollama": [7, 9], "lama": 7, "ui": 7, "lm": 7, "studio": 7, "jan": 7, "webui": 7, "openwebui": 7, "effect": 7, "level": 7, "hardwar": 7, "takeawai": [7, 8], "risk": 8, "ai": 8, "amplifi": 8, "exist": 8, "harm": 8, "novel": 8, "associ": 8, "autonom": 8, "exacerb": 8, "factor": 8, "specif": 8, "guidanc": 8, "govern": 8, "organ": 8, "privat": 8, "sector": 8, "openai": 8, "anthrop": 8, "googl": 8, "rubric": 8, "mlcommon": 8, "centr": 8, "porquoi": 8, "red": 8, "team": 8, "constitut": 8, "explain": 8, "xai": 8, "plan": 8, "phase": 8, "definit": 8, "research": [8, 9], "identif": 8, "architectur": 8, "select": 8, "go": 8, "market": 8, "compon": 8, "salad": 8, "bench": 8, "truthfulqa": 8, "harmbench": 8, "safebench": 8, "techniqu": [8, 9], "repres": 8, "layer": 8, "map": 8, "rule": 8, "filter": 8, "moder": 8, "bad": 8, "good": 8, "guard": 8, "judg": 8, "valid": 8, "engin": 9, "json": 9, "mode": 9, "logit": 9, "process": 9, "outlin": 9, "langchain": 9, "best": 9, "compar": 9, "solut": 9, "ongo": 9, "debat": 9, "acknowledg": 9}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"About the Book": [[0, "about-the-book"], [2, "about-the-book"]], "Contents": [[0, "contents"], [3, "contents"], [4, "contents"], [5, "contents"], [6, "contents"], [7, "contents"], [8, "contents"], [9, "contents"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Practical Approach": [[0, "a-practical-approach"]], "An Open Source Approach": [[0, "an-open-source-approach"]], "Open Source Book": [[0, "open-source-book"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Outcomes": [[0, "outcomes"]], "Prerequisites": [[0, "prerequisites"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "Code Repository": [[0, "code-repository"]], "Python Environment Setup": [[0, "python-environment-setup"]], "API Keys Configuration": [[0, "api-keys-configuration"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "About the Author": [[0, "about-the-author"]], "Preface": [[1, "preface"], [2, "preface"]], "Taming LLMs": [[2, "taming-llms"]], "A Practical Guide to LLM Pitfalls with Open Source Software": [[2, "a-practical-guide-to-llm-pitfalls-with-open-source-software"]], "Chapter 1: The Evals Gap": [[2, "chapter-1-the-evals-gap"]], "Chapter 2: Managing Input Data": [[2, "chapter-2-managing-input-data"]], "Chapter 3: Structured Output": [[2, "chapter-3-structured-output"]], "Chapter 4: Safety": [[2, "chapter-4-safety"]], "Chapter 5: Preference-Based Alignment": [[2, "chapter-5-preference-based-alignment"]], "Chapter 6: Local LLMs in Practice": [[2, "chapter-6-local-llms-in-practice"]], "Chapter 7: The Falling Cost Paradox": [[2, "chapter-7-the-falling-cost-paradox"]], "Chapter 8: Frontiers": [[2, "chapter-8-frontiers"]], "Appendix A: Tools and Resources": [[2, "appendix-a-tools-and-resources"]], "Preference-Based Alignment": [[3, "preference-based-alignment"]], "Introduction": [[3, "introduction"], [5, "introduction"], [6, "introduction"], [7, "introduction"], [8, "introduction"], [9, "introduction"]], "From Raw Capabilities to Preference Alignment": [[3, "from-raw-capabilities-to-preference-alignment"]], "On the Misalignment of Language Models": [[3, "on-the-misalignment-of-language-models"]], "Aligning Language Models with Human Preferences": [[3, "aligning-language-models-with-human-preferences"]], "Supervised Fine-Tuning (SFT) for Model Alignment": [[3, "supervised-fine-tuning-sft-for-model-alignment"]], "Augmenting SFT with Human Preferences": [[3, "augmenting-sft-with-human-preferences"]], "Is Post-Training the Answer?": [[3, "is-post-training-the-answer"]], "Limitations": [[3, "limitations"]], "Model Collapse": [[3, "model-collapse"]], "Faking Alignment": [[3, "faking-alignment"]], "Case Study: Aligning a Language Model to a Policy": [[3, "case-study-aligning-a-language-model-to-a-policy"]], "Experimental Setup": [[3, "experimental-setup"]], "Deliverables": [[3, "deliverables"]], "A Note on smolLM2 Models": [[3, "a-note-on-smollm2-models"]], "Policy": [[3, "policy"]], "Preference Dataset - Synthetic Dataset Generation": [[3, "preference-dataset-synthetic-dataset-generation"]], "User Prompts": [[3, "user-prompts"]], "Rejected Responses": [[3, "rejected-responses"]], "Chosen Responses": [[3, "chosen-responses"]], "Generate DPO Dataset": [[3, "generate-dpo-dataset"]], "DPO-Based Optimization": [[3, "dpo-based-optimization"]], "Data Preparation": [[3, "data-preparation"]], "Fine-Tuning": [[3, "fine-tuning"]], "Vibe Check": [[3, "vibe-check"]], "Alignment Evaluation": [[3, "alignment-evaluation"]], "Discussion and Conclusions": [[3, "discussion-and-conclusions"]], "References": [[3, "references"], [4, "references"], [5, "references"], [6, "references"], [7, "references"], [8, "references"], [9, "references"]], "The Falling Cost Paradox": [[4, "the-falling-cost-paradox"]], "Why Optimization Matters More Than Ever": [[4, "why-optimization-matters-more-than-ever"]], "Right-Sizing LLMs: A Strategic Approach": [[4, "right-sizing-llms-a-strategic-approach"]], "Metrics": [[4, "metrics"], [5, "metrics"]], "Requirements": [[4, "requirements"]], "Business Requirements": [[4, "business-requirements"]], "Performance Requirements": [[4, "performance-requirements"]], "Operational Requirements": [[4, "operational-requirements"]], "Technical Requirements": [[4, "technical-requirements"]], "Quantization": [[4, "quantization"], [7, "quantization"]], "Check-list": [[4, "check-list"]], "Conclusion": [[4, "conclusion"], [5, "conclusion"], [6, "conclusion"], [7, "conclusion"], [8, "conclusion"], [9, "conclusion"]], "The Evals Gap": [[5, "the-evals-gap"]], "Non-Deterministic Generative Machines": [[5, "non-deterministic-generative-machines"]], "Emerging Properties": [[5, "emerging-properties"]], "Problem Statement": [[5, "problem-statement"], [9, "problem-statement"]], "Evals of Traditional Software vs LLMs": [[5, "evals-table"]], "Evals Design": [[5, "evals-design"]], "LLM Application Testing Requirements Matrix": [[5, "validation-requirements"]], "Conceptual Overview": [[5, "conceptual-overview"]], "Design Considerations": [[5, "design-considerations"]], "Key Metrics for Evaluating Generative Tasks": [[5, "key-metrics"]], "Evaluators": [[5, "evaluators"]], "Model-Based Evaluation": [[5, "model-based-evaluation"]], "Evaluating Evaluators": [[5, "evaluating-evaluators"]], "Benchmarks and Leaderboards": [[5, "benchmarks-and-leaderboards"]], "Tools": [[5, "tools"], [9, "tools"]], "LightEval": [[5, "lighteval"]], "MMLU Econometrics Task Dataset sample": [[5, "mmlu-econometrics"]], "Model Families Evaluated Using LightEval": [[5, "model-families"]], "LangSmith": [[5, "langsmith"]], "PromptFoo": [[5, "promptfoo"]], "Comparison": [[5, "comparison"], [7, "comparison"], [7, "id37"]], "Comparison of Lighteval, LangSmith, and Promptfoo": [[5, "tool-comparison"]], "Managing Input Data": [[6, "managing-input-data"]], "Parsing Documents": [[6, "parsing-documents"]], "MarkItDown": [[6, "markitdown"]], "Docling": [[6, "docling"]], "Frameworks-Based Parsing": [[6, "frameworks-based-parsing"]], "Structured Data Extraction": [[6, "structured-data-extraction"]], "Retrieval-Augmented Generation": [[6, "retrieval-augmented-generation"]], "Case Studies": [[6, "case-studies"]], "Case Study I: Content Chunking with Contextual Linking": [[6, "case-study-i-content-chunking-with-contextual-linking"]], "Generating long-form content": [[6, "generating-long-form-content"]], "Discussion": [[6, "discussion"], [6, "id14"], [9, "discussion"]], "Case Study II: Github RAG": [[6, "case-study-ii-github-rag"]], "Case Study III: Quiz Generation with Citations": [[6, "case-study-iii-quiz-generation-with-citations"]], "Use Case": [[6, "use-case"]], "Implementation": [[6, "implementation"]], "Example Usage": [[6, "example-usage"]], "Local LLMs in Practice": [[7, "local-llms-in-practice"]], "Choosing your Model": [[7, "choosing-your-model"]], "Task Suitability": [[7, "task-suitability"]], "Benchmark results for Llama 2 family of models.": [[7, "llama2-benchmark"]], "Performance & Cost": [[7, "performance-cost"]], "Licensing": [[7, "licensing"]], "Open Source LLMs.": [[7, "open-source-llms"]], "Community Support": [[7, "community-support"]], "Customization": [[7, "customization"]], "Mistral fine-tuning costs as of December 22, 2024.": [[7, "mistral-costs"]], "Tools for Local LLM Deployment": [[7, "tools-for-local-llm-deployment"]], "Serving Models": [[7, "serving-models"]], "LLama.cpp": [[7, "llama-cpp"]], "Llamafile": [[7, "llamafile"]], "Ollama": [[7, "ollama"], [9, "ollama"]], "lama.cpp vs Ollama vs Llamafile Comparison": [[7, "feature-comparison-local"]], "UI": [[7, "ui"]], "LM Studio": [[7, "lm-studio"]], "Jan": [[7, "jan"]], "Open WebUI": [[7, "open-webui"]], "LM Studio vs Jan vs OpenWebUI Comparison": [[7, "feature-comparison-ui"]], "Case Study: The Effect of Quantization on LLM Performance": [[7, "case-study-the-effect-of-quantization-on-llm-performance"]], "Prompts Dataset": [[7, "prompts-dataset"]], "Quantization Levels": [[7, "quantization-levels"]], "Benchmarking": [[7, "benchmarking"], [8, "benchmarking"]], "Results": [[7, "results"]], "Quantization Benchmarks": [[7, "quantization-benchmarks"]], "Benchmarking Hardware": [[7, "benchmarking-hardware"]], "Takeaways": [[7, "takeaways"], [8, "takeaways"]], "Safety": [[8, "safety"]], "Safety Risks": [[8, "safety-risks"]], "General AI Safety Risks": [[8, "general-ai-safety-risks"]], "Amplified Existing Harms and Novel Risks": [[8, "amplified-existing-harms-and-novel-risks"]], "Risks Associated with Autonomous AI": [[8, "risks-associated-with-autonomous-ai"]], "Exacerbating Factors": [[8, "exacerbating-factors"]], "LLMs Specific Safety Risks": [[8, "llms-specific-safety-risks"]], "Guidance": [[8, "guidance"]], "Governments & Organizations": [[8, "governments-organizations"]], "Private Sector": [[8, "private-sector"]], "OpenAI": [[8, "openai"]], "Anthropic": [[8, "anthropic"]], "Google": [[8, "google"]], "Rubrics": [[8, "rubrics"]], "MLCommons AI Safety Benchmark": [[8, "mlcommons-ai-safety-benchmark"]], "Centre for the Governance of AI Rubric": [[8, "centre-for-the-governance-of-ai-rubric"]], "Porquoi": [[8, "porquoi"]], "Approaches": [[8, "approaches"]], "Red Teaming": [[8, "red-teaming"]], "Constitutional AI": [[8, "constitutional-ai"]], "Explainable AI (XAI)": [[8, "explainable-ai-xai"]], "Designing a Safety Plan": [[8, "designing-a-safety-plan"]], "Phase 1. Policy Definition": [[8, "phase-1-policy-definition"]], "Phase 2. User Research & Risk Identification": [[8, "phase-2-user-research-risk-identification"]], "Phase 3. Evaluation Framework": [[8, "phase-3-evaluation-framework"]], "Phase 4. Safety Architecture Design": [[8, "phase-4-safety-architecture-design"]], "Phase 5. Implementation & Tools Selection": [[8, "phase-5-implementation-tools-selection"]], "Phase 6. Go-to-Market": [[8, "phase-6-go-to-market"]], "Common Pitfalls": [[8, "common-pitfalls"]], "Technical Implementation Components": [[8, "technical-implementation-components"]], "Benchmarks & Datasets": [[8, "benchmarks-datasets"]], "SALAD-Bench": [[8, "salad-bench"]], "TruthfulQA": [[8, "truthfulqa"]], "HarmBench": [[8, "harmbench"]], "SafeBench": [[8, "safebench"]], "Tools & Techniques": [[8, "tools-techniques"]], "Representative Safety Layer Risk Map.": [[8, "safety-layer-table"]], "Rules-Based Safety Filtering": [[8, "rules-based-safety-filtering"]], "Rules-Based Safety Filtering Tools.": [[8, "safety-layer-tools"]], "LLM-Based Safety Filtering": [[8, "llm-based-safety-filtering"]], "Custom Moderation": [[8, "custom-moderation"]], "Case Study: Implementing a Safety Filter": [[8, "case-study-implementing-a-safety-filter"]], "Evals Dataset": [[8, "evals-dataset"]], "Bad Samples": [[8, "bad-samples"]], "Good Samples": [[8, "good-samples"]], "Safety Filters": [[8, "safety-filters"]], "LLM-Guard": [[8, "llm-guard"]], "Mistral Moderation API": [[8, "mistral-moderation-api"]], "OpenAI Moderation API": [[8, "openai-moderation-api"]], "Custom Judge Validator": [[8, "custom-judge-validator"]], "Structured Output": [[9, "structured-output"]], "Techniques": [[9, "techniques"]], "Prompt Engineering": [[9, "prompt-engineering"]], "JSON Mode (Fine-Tuned)": [[9, "json-mode-fine-tuned"]], "Logit Post-Processing": [[9, "logit-post-processing"]], "Outlines": [[9, "outlines"]], "LangChain": [[9, "langchain"]], "Best Practices": [[9, "best-practices"]], "Comparing Solutions": [[9, "comparing-solutions"]], "Structured Output Frameworks Comparison": [[9, "structured-output-frameworks"]], "Research and Ongoing Debate": [[9, "research-and-ongoing-debate"]], "Acknowledgements": [[9, "acknowledgements"]]}, "indexentries": {}})
\ No newline at end of file
+Search.setIndex({"docnames": ["markdown/intro", "markdown/preface", "markdown/toc", "notebooks/alignment", "notebooks/cost", "notebooks/evals", "notebooks/input", "notebooks/local", "notebooks/safety", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/preface.md", "markdown/toc.md", "notebooks/alignment.ipynb", "notebooks/cost.ipynb", "notebooks/evals.ipynb", "notebooks/input.ipynb", "notebooks/local.ipynb", "notebooks/safety.ipynb", "notebooks/structured_output.ipynb"], "titles": ["2. About the Book", "1. Preface", "Taming LLMs", "7. Preference-Based Alignment", "9. The Falling Cost Paradox", "3. The Evals Gap", "5. Managing Input Data", "8. Local LLMs in Practice", "6. Safety", "4. Structured Output"], "terms": {"am": [0, 8], "alwai": [0, 3, 4, 5, 6, 9], "do": [0, 3, 4, 5, 6, 7, 8, 9], "which": [0, 3, 4, 5, 6, 7, 8, 9], "cannot": [0, 3, 4, 5, 7, 8], "order": [0, 3, 5, 6, 8, 9], "mai": [0, 1, 3, 4, 5, 6, 7, 8, 9], "learn": [0, 3, 5, 6, 7, 8, 9], "how": [0, 1, 3, 4, 5, 6, 7, 8, 9], "pablo": [0, 5], "picasso": 0, "In": [0, 3, 4, 5, 6, 7, 8, 9], "recent": [0, 3, 4, 5, 6, 7, 8, 9], "year": [0, 2, 3, 4, 5, 6, 7, 8, 9], "larg": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "languag": [0, 1, 2, 4, 5, 6, 7, 8, 9], "model": [0, 1, 2, 4, 6, 8, 9], "llm": [0, 1, 3, 6, 9], "have": [0, 1, 3, 4, 5, 6, 7, 8, 9], "emerg": [0, 3, 4, 6, 7, 8, 9], "transform": [0, 1, 3, 5, 6, 7, 8, 9], "forc": [0, 5, 9], "technologi": [0, 1, 4, 5, 6, 7, 8], "promis": [0, 3, 4, 5, 8], "revolution": [0, 8], "build": [0, 2, 3, 5, 6, 7, 8, 9], "product": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "interact": [0, 3, 4, 5, 6, 7, 8, 9], "comput": [0, 3, 4, 5, 6, 7, 8, 9], "from": [0, 1, 4, 5, 6, 7, 8, 9], "chatgpt": [0, 3, 4, 7, 9], "llama": [0, 3, 4, 5, 8, 9], "github": [0, 2, 3, 4, 5, 7, 8, 9], "copilot": 0, "claud": [0, 3, 5, 7, 8], "artifact": 0, "system": [0, 3, 4, 5, 6, 7, 8, 9], "captur": [0, 1, 3, 5, 7, 8], "public": [0, 3, 5, 7, 8], "imagin": [0, 7], "spark": 0, "gold": [0, 3, 6, 8], "rush": 0, "ai": [0, 3, 4, 5, 7, 9], "power": [0, 2, 3, 4, 5, 7, 8, 9], "applic": [0, 1, 2, 3, 4, 6, 7, 8, 9], "howev": [0, 3, 4, 5, 6, 7, 8, 9], "beneath": 0, "surfac": [0, 5], "technolog": [0, 1, 4, 5, 6, 8], "revolut": [0, 4], "li": [0, 3, 5, 7, 8, 9], "complex": [0, 1, 3, 5, 6, 7, 8, 9], "landscap": [0, 3, 5, 7], "softwar": [0, 1, 3, 4, 6, 7, 8, 9], "develop": [0, 1, 3, 4, 5, 6, 7, 8, 9], "tech": [0, 7, 8], "leader": [0, 2, 5, 8], "must": [0, 3, 4, 5, 7, 8, 9], "navig": [0, 2, 5, 7, 8], "focus": [0, 3, 4, 5, 6, 7, 8, 9], "bring": [0, 3, 7], "awar": [0, 3, 4, 5, 6, 8], "limit": [0, 1, 2, 4, 5, 6, 7, 8, 9], "har": [0, 2, 5], "solut": [0, 2, 4, 5, 6, 7, 8], "overcom": [0, 5, 6], "them": [0, 1, 3, 4, 5, 6, 7, 8, 9], "robust": [0, 3, 4, 5, 6, 7, 8, 9], "It": [0, 3, 4, 5, 6, 7, 8, 9], "offer": [0, 3, 4, 5, 6, 7, 8, 9], "critic": [0, 2, 3, 4, 5, 6, 7, 8, 9], "implement": [0, 2, 3, 4, 5, 7, 9], "back": [0, 5, 6, 7, 8, 9], "reproduc": [0, 1, 2, 5, 7], "exampl": [0, 1, 2, 3, 5, 7, 8, 9], "while": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "mani": [0, 1, 3, 4, 5, 6, 7, 8, 9], "resourc": [0, 3, 4, 5, 6, 7, 8], "cover": [0, 3, 4, 5, 6, 7, 8, 9], "capabl": [0, 1, 2, 4, 5, 6, 7, 8, 9], "specif": [0, 3, 4, 5, 6, 7, 9], "hidden": [0, 3, 8], "pitfal": [0, 1, 3, 4, 5, 6, 7, 9], "engin": [0, 1, 2, 3, 4, 5, 7, 8], "technic": [0, 1, 2, 3, 5, 7, 9], "face": [0, 3, 4, 5, 6, 7, 8], "when": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "comprehens": [0, 2, 3, 4, 5, 6, 7, 8, 9], "guid": [0, 1, 3, 4, 5, 6, 7, 8, 9], "leverag": [0, 3, 5, 6, 7, 8, 9], "battl": [0, 2, 7], "test": [0, 2, 3, 4, 6, 7, 8, 9], "tool": [0, 1, 3, 4, 6], "throughout": [0, 4, 5, 6, 7, 8], "tackl": [0, 3, 5, 6, 8], "follow": [0, 3, 4, 5, 6, 7, 8, 9], "non": [0, 3, 7, 8, 9], "exhaust": [0, 7], "list": [0, 3, 5, 6, 7, 8, 9], "structur": [0, 3, 4, 5, 7, 8], "un": 0, "reliabl": [0, 1, 3, 4, 5, 7, 8, 9], "struggl": [0, 1, 3, 5, 6, 7, 8, 9], "maintain": [0, 1, 3, 4, 5, 6, 7, 8, 9], "consist": [0, 1, 3, 4, 5, 6, 7, 8, 9], "output": [0, 1, 3, 5, 6, 7, 8], "format": [0, 3, 4, 5, 6, 7, 8, 9], "complic": [0, 8], "integr": [0, 1, 3, 4, 5, 6, 7, 8, 9], "larger": [0, 3, 4, 5, 6, 7, 8, 9], "make": [0, 3, 4, 5, 6, 7, 8, 9], "error": [0, 3, 5, 6, 8, 9], "handl": [0, 3, 4, 5, 6, 7, 8, 9], "more": [0, 1, 3, 5, 6, 7, 8, 9], "input": [0, 3, 5, 7, 8, 9], "data": [0, 1, 4, 5, 7, 8, 9], "manag": [0, 1, 4, 5, 7, 8, 9], "ar": [0, 1, 3, 4, 5, 6, 7, 8, 9], "sensit": [0, 3, 4, 5, 6, 7, 8], "oper": [0, 3, 5, 6, 7, 8, 9], "stale": [0, 6], "long": [0, 1, 3, 4, 5, 7, 8, 9], "context": [0, 1, 3, 4, 5, 6, 7, 8, 9], "requir": [0, 3, 6, 7, 8, 9], "care": [0, 3, 4, 5, 6, 7, 8, 9], "retriev": [0, 4, 5, 7], "strategi": [0, 3, 4, 5, 6, 7, 8, 9], "tradit": [0, 3, 7, 8], "methodologi": [0, 3, 5, 7, 8, 9], "break": [0, 1, 3, 4, 5, 6, 8], "down": [0, 1, 4, 5, 6, 7, 8], "deal": [0, 3, 7], "determinist": [0, 9], "gener": [0, 1, 4, 7, 9], "new": [0, 2, 3, 4, 5, 6, 7, 8, 9], "safeti": [0, 3, 5, 9], "align": [0, 4, 5, 6, 7, 8, 9], "can": [0, 1, 3, 4, 5, 6, 7, 8, 9], "harm": [0, 3, 5, 7], "bias": [0, 3, 5, 7, 8, 9], "inappropri": [0, 3, 8], "safeguard": [0, 5, 8], "monitor": [0, 3, 4, 5, 6, 7, 8], "ensur": [0, 3, 4, 5, 6, 7, 8, 9], "safe": [0, 3, 5, 8, 9], "deploy": [0, 3, 4, 5, 8, 9], "vendor": [0, 4, 5, 7], "lock": [0, 3, 4, 7], "cloud": [0, 3, 4, 5, 7, 8, 9], "base": [0, 1, 4, 7, 9], "provid": [0, 2, 3, 4, 5, 6, 7, 8, 9], "creat": [0, 1, 3, 4, 5, 6, 7, 8, 9], "signific": [0, 3, 4, 5, 6, 7, 8, 9], "depend": [0, 3, 4, 5, 6, 7, 9], "through": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "proprietari": [0, 3, 7, 8, 9], "infrastructur": [0, 4, 7], "difficult": [0, 3, 5, 8], "switch": [0, 7], "self": [0, 3, 5, 6, 7, 8, 9], "host": [0, 4, 5, 7, 8], "cost": [0, 3, 5, 6, 8, 9], "optim": [0, 1, 5, 6, 7, 8], "The": [0, 1, 3, 6, 8, 9], "financi": [0, 1, 3, 4, 5, 6, 8, 9], "quickli": [0, 3, 4, 7], "becom": [0, 3, 4, 5, 6, 7, 8, 9], "prohibit": [0, 3, 5, 7], "without": [0, 1, 3, 4, 5, 7, 8, 9], "take": [0, 2, 3, 4, 5, 6, 7, 8, 9], "hand": [0, 7, 8, 9], "focu": [0, 2, 3, 4, 5, 6, 7, 8, 9], "access": [0, 3, 4, 5, 6, 7, 8, 9], "all": [0, 1, 3, 4, 5, 6, 7, 8, 9], "fulli": [0, 3, 5, 8], "document": [0, 3, 4, 5, 7, 8, 9], "allow": [0, 5, 6, 7, 8, 9], "reader": [0, 2, 6], "replic": [0, 5, 8, 9], "result": [0, 3, 4, 5, 6, 8, 9], "exactli": [0, 5, 9], "design": [0, 1, 3, 7, 9], "run": [0, 3, 4, 5, 6, 7, 8, 9], "consum": [0, 3, 4, 5, 6, 7, 8, 9], "grade": [0, 3, 4, 5, 6, 7, 8], "hardwar": [0, 3, 4, 5], "expens": [0, 3, 4, 5, 6, 7, 8], "avail": [0, 3, 4, 5, 6, 7, 8, 9], "notebook": [0, 3, 9], "modifi": [0, 3, 5, 8, 9], "extend": [0, 3, 4, 5, 6, 7, 9], "minim": [0, 3, 4, 5, 7, 8, 9], "effect": [0, 1, 3, 4, 5, 6, 8, 9], "framework": [0, 3, 4, 5, 7], "wai": [0, 3, 4, 5, 6, 7, 8, 9], "priorit": [0, 3, 5, 6, 7, 8], "transpar": [0, 3, 4, 5, 7, 8], "visibl": [0, 5], "being": [0, 3, 4, 5, 6, 7, 8, 9], "better": [0, 2, 3, 4, 5, 6, 7, 8, 9], "understand": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "custom": [0, 3, 5, 6, 9], "flexibl": [0, 4, 5, 7, 8, 9], "adapt": [0, 3, 4, 5, 7, 8], "us": [0, 1, 3, 4, 7, 8, 9], "case": [0, 4, 5, 9], "unlik": [0, 3, 5, 7], "black": [0, 3], "box": [0, 7], "commerci": [0, 3, 5, 7, 8, 9], "most": [0, 3, 4, 5, 6, 7, 8, 9], "freeli": [0, 9], "foster": [0, 3, 5, 8, 9], "reduc": [0, 3, 4, 5, 6, 7, 8, 9], "independ": [0, 5, 6, 8, 9], "freedom": [0, 7, 9], "architectur": [0, 3, 4, 5, 7, 9], "decis": [0, 3, 4, 5, 6, 7, 8], "keep": [0, 3, 5, 6, 7, 8], "principl": [0, 3, 5, 7, 8], "itself": [0, 3, 5, 6, 7, 8], "": [0, 1, 3, 4, 5, 6, 7, 8, 9], "live": [0, 1, 5, 6, 8], "evolv": [0, 3, 4, 5, 7, 8], "chang": [0, 3, 5, 6, 7, 8], "encourag": [0, 3, 5, 8, 9], "report": [0, 3, 5, 6, 7, 8, 9], "suggest": [0, 3, 5, 7, 8, 9], "improv": [0, 3, 4, 5, 6, 7, 8, 9], "contribut": [0, 4, 5, 6, 7, 8], "via": [0, 3, 4, 5, 6, 7, 8, 9], "pull": [0, 7], "request": [0, 3, 4, 5, 6, 7, 8, 9], "share": [0, 3, 5, 6, 7, 8, 9], "own": [0, 3, 4, 5, 6, 7, 8], "experi": [0, 3, 4, 5, 6, 7, 8, 9], "commun": [0, 3, 4, 5, 6, 8, 9], "propos": [0, 4, 5, 6, 8], "chapter": [0, 3, 4, 5, 6, 7, 8, 9], "section": [0, 3, 4, 5, 6, 7, 8, 9], "found": [0, 3, 4, 5, 7, 9], "http": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "com": [0, 2, 3, 4, 5, 6, 7, 8, 9], "souzatharsi": [0, 2, 3, 4, 5, 6, 7, 8, 9], "tamingllm": [0, 2, 3, 4, 5, 6, 7, 8, 9], "whether": [0, 3, 4, 5, 6, 7, 8, 9], "you": [0, 1, 3, 4, 5, 6, 7, 8, 9], "ve": [0, 7], "typo": [0, 8], "want": [0, 1, 3, 6, 7, 8, 9], "welcom": 0, "pleas": [0, 3, 5, 7, 8], "feel": [0, 7], "free": [0, 1, 3, 5, 7, 8], "look": [0, 2, 3, 4, 5, 6, 7, 8], "our": [0, 1, 3, 4, 5, 6, 7, 8, 9], "goal": [0, 1, 3, 5, 6, 8, 9], "discourag": [0, 6], "enabl": [0, 3, 4, 5, 6, 7, 8, 9], "By": [0, 1, 2, 3, 5, 6, 8, 9], "upfront": [0, 2, 4], "equip": [0, 2, 5, 8], "avoid": [0, 3, 5, 7, 8, 9], "current": [0, 2, 3, 4, 5, 6, 8, 9], "discours": [0, 2], "around": [0, 2, 3, 5, 6, 7, 8, 9], "tend": [0, 2, 5, 8], "toward": [0, 3, 5, 8, 9], "extrem": [0, 3, 4, 5, 8], "either": [0, 3, 5, 6, 7, 8], "uncrit": 0, "enthusiasm": 0, "wholesal": [0, 5], "dismiss": 0, "differ": [0, 3, 4, 5, 6, 7, 8, 9], "rather": [0, 1, 3, 4, 5, 6, 7, 8], "than": [0, 1, 3, 5, 6, 7, 8, 9], "theoret": [0, 3], "examin": [0, 3, 5, 6, 7, 8, 9], "first": [0, 1, 3, 4, 5, 6, 7, 8, 9], "everi": [0, 4, 5, 8], "concept": [0, 3, 5, 8], "illustr": [0, 3, 5, 6, 7, 8, 9], "execut": [0, 5, 7, 8], "immedi": [0, 3, 4, 5, 7], "analysi": [0, 1, 3, 4, 5, 6, 7, 8], "balanc": [0, 3, 4, 5, 7, 8, 9], "both": [0, 3, 4, 5, 6, 7, 8], "help": [0, 3, 4, 5, 6, 7, 8, 9], "inform": [0, 3, 4, 5, 6, 7, 8, 9], "lead": [0, 1, 3, 4, 5, 6, 7, 8, 9], "genai": [0, 1, 3, 6, 8], "initi": [0, 1, 3, 4, 5, 6, 7, 8, 9], "advoc": [0, 8], "anyon": [0, 8], "seek": [0, 5, 7, 8], "work": [0, 1, 3, 4, 5, 6, 7, 8, 9], "typic": [0, 3, 4, 5, 7, 8, 9], "job": [0, 5, 6, 7, 8], "role": [0, 3, 5, 6, 7, 8, 9], "platform": [0, 5, 6, 7, 8, 9], "backend": [0, 3, 5], "exist": [0, 3, 4, 5, 7], "ml": [0, 6, 8], "transit": [0, 4, 5, 7, 9], "overse": 0, "motiv": [0, 3, 4, 5, 6, 9], "need": [0, 3, 4, 5, 6, 7, 8, 9], "readi": [0, 5, 8], "desir": [0, 3, 5, 9], "perform": [0, 3, 5, 6, 8, 9], "earli": [0, 3, 4, 5, 8, 9], "befor": [0, 3, 4, 5, 6, 8, 9], "thei": [0, 1, 3, 4, 5, 6, 7, 8, 9], "costli": [0, 5, 6, 8], "problem": [0, 1, 2, 3, 4, 6, 7, 8], "too": [0, 1, 3, 5, 6, 7, 8], "late": [0, 3, 4, 8, 9], "lifecycl": [0, 7, 8], "after": [0, 1, 3, 5, 6, 7, 8, 9], "read": [0, 3, 4, 5, 6, 8, 9], "implic": [0, 1, 3, 5, 8], "recommend": [0, 3, 5, 6, 7, 8, 9], "abl": [0, 3, 5, 9], "deploi": [0, 3, 5, 7, 8], "proper": [0, 3, 4, 7, 8, 9], "realist": [0, 3, 4, 8], "effort": [0, 5, 7, 8, 9], "estim": [0, 4, 5, 6, 8], "project": [0, 3, 4, 5, 6, 7, 8], "impact": [0, 3, 4, 5, 6, 7, 8, 9], "timelin": 0, "To": [0, 3, 5, 7, 8, 9], "should": [0, 3, 4, 5, 6, 7, 8, 9], "basic": [0, 3, 5, 7, 8], "program": [0, 5, 6, 7, 9], "knowledg": [0, 3, 5, 6, 7, 8], "mistral": [0, 3, 9], "openai": [0, 3, 5, 6, 7, 9], "anthrop": [0, 3, 6, 9], "similar": [0, 3, 4, 5, 6, 7, 9], "dive": [0, 4], "here": [0, 2, 3, 4, 5, 6, 7, 8, 9], "get": [0, 3, 4, 5, 6, 7, 8, 9], "start": [0, 3, 4, 5, 6, 7, 8, 9], "clone": [0, 3], "companion": 0, "git": 0, "cd": 0, "activ": [0, 3, 4, 5, 6, 7, 8], "virtual": [0, 5], "m": [0, 3, 5, 6, 7, 8, 9], "venv": [0, 9], "tame": [0, 3, 4, 5, 6, 7, 8, 9], "env": [0, 3, 5, 6, 8, 9], "bin": [0, 7], "On": [0, 5, 7, 9], "window": [0, 4, 5, 6, 7], "script": [0, 7], "try": [0, 1, 3, 5, 8, 9], "each": [0, 3, 4, 5, 6, 7, 8, 9], "contain": [0, 3, 4, 5, 6, 7, 8, 9], "possibl": [0, 3, 4, 5, 6, 7, 8, 9], "includ": [0, 1, 3, 4, 5, 6, 7, 8, 9], "necessari": [0, 3, 4, 5, 8], "instal": [0, 3, 5, 7, 9], "go": [0, 3, 5, 6, 9], "prefer": [0, 5, 6, 7, 8, 9], "packag": [0, 4, 5, 6, 7, 9], "e": [0, 1, 3, 4, 5, 6, 7, 8, 9], "g": [0, 3, 4, 5, 6, 7, 8, 9], "pip": [0, 3, 5, 7, 9], "poetri": [0, 8], "file": [0, 3, 5, 6, 7, 8, 9], "root": [0, 3], "directori": [0, 5, 7], "add": [0, 3, 5, 6, 7, 8], "other": [0, 3, 4, 5, 6, 7, 8, 9], "openai_api_kei": [0, 3], "your_openai_api_key_her": 0, "never": [0, 9], "commit": [0, 3, 5, 8], "version": [0, 3, 4, 5, 7, 8, 9], "control": [0, 1, 3, 4, 5, 6, 7, 8, 9], "kept": [0, 5], "privat": [0, 5], "If": [0, 1, 3, 4, 5, 7, 8, 9], "encount": [0, 2, 5, 8], "rate": [0, 3, 4, 5, 6, 7, 8], "consid": [0, 3, 4, 5, 6, 7, 8, 9], "smaller": [0, 3, 4, 5, 6, 7, 9], "retri": [0, 9], "logic": [0, 1, 3, 5, 6, 8], "conflict": [0, 3, 5], "fresh": 0, "like": [0, 1, 3, 4, 5, 6, 7, 8, 9], "check": [0, 5, 6, 7, 8, 9], "page": [0, 5, 6, 7], "known": [0, 5, 6, 8, 9], "now": [0, 1, 3, 4, 5, 6, 7, 8, 9], "let": [0, 3, 4, 5, 6, 7, 8, 9], "begin": [0, 5, 7, 8, 9], "explor": [0, 1, 3, 4, 5, 6, 7, 8, 9], "tharsi": [0, 2, 3, 4, 5, 6, 7, 8, 9], "souza": [0, 2, 3, 4, 5, 6, 7, 8, 9], "ph": [0, 8], "d": [0, 3, 4, 5, 6, 7, 8, 9], "scienc": [0, 3, 5, 8], "ucl": 0, "univers": [0, 5, 7, 8], "london": 0, "scientist": [0, 1, 7, 8], "special": [0, 4, 5, 6, 7, 8, 9], "he": [0, 3, 5, 8], "lectur": 0, "columbia": 0, "master": [0, 4, 7, 9], "appli": [0, 3, 5, 6, 7, 8, 9], "analyt": 0, "incom": [0, 5, 6], "head": [0, 3, 5, 6, 8, 9], "equiti": [0, 5, 6], "citadel": 0, "former": [0, 1, 5, 7], "senior": [0, 5], "vp": 0, "two": [0, 3, 4, 5, 6, 7, 8, 9], "sigma": [0, 3], "invest": [0, 3, 4, 5, 6, 8], "mentor": 0, "under": [0, 3, 4, 5, 7, 8, 9], "repres": [0, 3, 4, 5, 6, 7, 9], "student": [0, 3, 6, 8], "profession": [0, 3, 5, 8, 9], "divers": [0, 3, 4, 5, 6, 8], "global": [0, 5, 6, 8], "ai1": 0, "ecosystem": [0, 4, 5, 7], "With": [0, 3, 5, 7, 8, 9], "over": [0, 2, 3, 4, 5, 6, 7, 8, 9], "15": [0, 5, 6, 7, 8, 9], "deliv": [0, 4, 5, 7], "across": [0, 1, 3, 4, 5, 6, 7, 8, 9], "startup": 0, "fortun": 0, "500": [0, 3, 5, 6, 8], "compani": [0, 3, 4, 5, 6, 8, 9], "also": [0, 3, 4, 5, 6, 7, 8, 9], "numer": [0, 4, 5, 8, 9], "scholarli": 0, "frequent": [0, 5, 7, 9], "speaker": [0, 5], "academ": [0, 3, 5, 8], "busi": [0, 5, 6, 7, 8], "confer": [0, 6, 9], "ground": [0, 3, 5, 7], "background": [0, 1, 5, 6, 7], "draw": [0, 3, 5, 8, 9], "scale": [0, 3, 4, 5, 6, 7, 8, 9], "stage": [0, 3, 8, 9], "major": [0, 3, 4, 5, 6, 7, 8, 9], "institut": [0, 5, 8], "well": [0, 3, 4, 5, 6, 7, 8, 9], "uniqu": [0, 3, 4, 5, 6, 7, 8, 9], "bridg": [0, 6, 7, 8], "gap": [0, 1, 3, 4, 7, 8], "between": [0, 1, 3, 4, 5, 6, 7, 8, 9], "potenti": [0, 1, 3, 4, 5, 6, 7, 8, 9], "next": [0, 1, 3, 4, 5, 6, 7, 8, 9], "tell": [1, 3, 8], "mere": [1, 5], "what": [1, 3, 4, 5, 6, 7, 8, 9], "someth": [1, 5, 7], "i": [1, 2, 4, 5, 7, 8, 9], "emanuel": [1, 3, 5, 8], "derman": 1, "an": [1, 2, 3, 4, 5, 6, 7, 8, 9], "altern": [1, 3, 4, 5, 6, 7, 8], "titl": [1, 2, 3, 4, 5, 6, 7, 8, 9], "thi": [1, 2, 3, 4, 5, 6, 7, 8, 9], "book": [1, 5, 6], "could": [1, 3, 4, 5, 6, 7, 8, 9], "been": [1, 3, 4, 5, 7, 8], "behav": 1, "badli": 1, "come": [1, 3, 5, 6, 7, 8, 9], "notic": [1, 3, 4, 5, 8, 9], "parallel": [1, 3, 5, 7], "semin": [1, 8], "2011": 1, "coincident": 1, "just": [1, 3, 4, 5, 6, 7, 8, 9], "caution": 1, "against": [1, 3, 4, 5, 7, 8], "treat": [1, 5, 8], "perfect": [1, 5, 7], "represent": [1, 5, 6, 7, 8], "realiti": [1, 6, 8], "aim": [1, 3, 4, 5, 6, 7, 8, 9], "highlight": [1, 3, 5, 6, 7, 8, 9], "practic": [1, 3, 4, 5, 6, 8], "physicist": 1, "goldman": 1, "sach": 1, "quant": 1, "scientif": [1, 3, 5, 7], "fail": [1, 3, 5, 8], "we": [1, 3, 4, 5, 6, 7, 8, 9], "mistak": [1, 8], "approxim": [1, 4, 5, 9], "full": [1, 3, 4, 5, 6, 7, 8, 9], "assumpt": [1, 5, 8], "core": [1, 4, 5, 6, 7, 8], "premis": [1, 7], "hi": [1, 5, 8, 9], "aspect": [1, 3, 5, 6, 8], "world": [1, 3, 4, 5, 6, 7, 8, 9], "inher": [1, 2, 3, 5, 8], "involv": [1, 3, 4, 5, 6, 7, 8, 9], "simplif": 1, "argu": [1, 4, 8, 9], "crise": 1, "2008": 1, "crash": 1, "occur": [1, 3, 5, 8], "partli": 1, "becaus": [1, 3, 5, 8], "peopl": [1, 3, 5, 7, 8], "put": [1, 5, 7], "much": [1, 3, 5, 6, 7], "faith": 1, "mathemat": [1, 5, 7, 9], "recogn": [1, 3, 5, 8], "human": [1, 4, 5, 6, 7, 8, 9], "behavior": [1, 3, 5, 7, 8], "market": [1, 4, 5, 6, 7, 9], "dynam": [1, 3, 5, 6, 8], "constraint": [1, 3, 4, 5, 6, 7, 8, 9], "hallucin": [1, 3, 5, 6, 8, 9], "fact": [1, 3, 5, 6, 8], "reason": [1, 3, 5, 6, 7, 8, 9], "Their": [1, 5, 9], "respons": [1, 4, 5, 6, 7, 8, 9], "often": [1, 3, 4, 5, 6, 7, 8, 9], "convinc": [1, 3], "probabilist": [1, 5, 9], "train": [1, 4, 5, 7, 8, 9], "true": [1, 3, 4, 5, 6, 8, 9], "even": [1, 3, 4, 5, 7, 8, 9], "though": [1, 3, 4, 5, 7, 8, 9], "insist": 1, "machin": [1, 3, 7, 8, 9], "todai": [1, 4, 7, 9], "grow": [1, 3, 5, 7, 8, 9], "pervas": [1, 8], "belief": [1, 7, 8], "solv": [1, 3, 4, 5, 7, 8, 9], "ani": [1, 3, 4, 5, 6, 7, 8, 9], "content": 1, "wish": [1, 5], "user": [1, 4, 5, 6, 7, 9], "moreov": 1, "were": [1, 3, 5, 7, 8, 9], "token": [1, 3, 4, 5, 6, 7, 8, 9], "predict": [1, 3, 5, 7, 8, 9], "chatbot": [1, 3, 5, 7, 8], "twist": [1, 8], "wrap": [1, 7, 9], "further": [1, 3, 4, 5, 6, 7, 8, 9], "daili": [1, 4, 7, 8], "life": [1, 5, 7, 8], "workflow": [1, 4, 5, 7, 8, 9], "affect": [1, 5, 6, 7, 8], "decid": [1, 3, 5, 6], "action": [1, 3, 5, 6, 8], "coupl": [1, 7], "lack": [1, 3, 5, 6, 8, 9], "pose": [1, 3, 5, 6, 8, 9], "risk": [1, 3, 4, 5, 6, 7], "still": [1, 4, 5, 7, 8], "figur": [1, 5, 7], "out": [1, 3, 4, 5, 6, 7, 8, 9], "serv": [1, 3, 4, 5, 6, 8, 9], "introductori": [1, 2], "practition": [1, 4, 5, 7, 9], "builder": [1, 7], "who": [1, 3, 5, 7, 8, 9], "remain": [1, 3, 4, 5, 6, 7, 8], "clear": [1, 3, 4, 5, 7, 8, 9], "ei": 1, "about": [1, 3, 4, 5, 6, 7, 8, 9], "therefor": [1, 3, 5, 7, 8], "end": [1, 3, 4, 5, 6, 7, 8, 9], "detail": [1, 3, 4, 5, 6, 7, 8, 9], "python": [1, 2, 5, 6, 7, 8, 9], "code": [1, 2, 3, 5, 6, 7, 8, 9], "diminish": [1, 3, 4, 5], "promot": [1, 3, 5, 8], "nuanc": [1, 3, 5, 6, 7, 8, 9], "acknowledg": [1, 5, 8], "within": [1, 3, 4, 5, 6, 7, 8, 9], "trustworthi": [1, 8], "taught": 1, "u": [1, 3, 5, 6, 8, 9], "step": [1, 3, 4, 5, 6, 7, 8, 9], "where": [1, 3, 4, 5, 6, 7, 8, 9], "der11": 1, "why": [1, 3, 5, 8, 9], "confus": [1, 4, 8], "illus": 1, "disast": [1, 5], "wall": [1, 7], "street": [1, 7], "press": [1, 5, 7], "isbn": [1, 3, 5], "9781439165010": 1, "url": [1, 2, 3, 4, 5, 6, 7, 8, 9], "googl": [1, 5, 7, 9], "co": [1, 3, 4, 5, 7, 8, 9], "uk": [1, 8], "id": [1, 5, 6, 7, 8, 9], "lke_cwm4wm8c": 1, "sign": [2, 5, 8], "up": [2, 3, 4, 5, 6, 7, 8], "receiv": [2, 3, 5, 6, 7, 8, 9], "updat": [2, 3, 4, 5, 6, 7, 8, 9], "abstract": [2, 5, 6, 8, 9], "heavili": [2, 3, 4, 5, 6, 8, 9], "gloss": 2, "fundament": [2, 3, 5, 7, 8, 9], "challeng": [2, 3, 4, 5, 6, 7, 8, 9], "convers": [2, 3, 4, 5, 6, 7, 8, 9], "kei": [2, 3, 4, 6, 7, 8, 9], "proven": [2, 4], "yet": [2, 3, 4, 5, 6, 7, 8, 9], "concret": [2, 4, 8, 9], "sidestep": 2, "misc": [2, 3, 4, 5, 6, 7, 8, 9], "tharsistpsouza2024tamingllm": [2, 3, 4, 5, 6, 7, 8, 9], "author": [2, 3, 4, 5, 6, 7, 8, 9], "t": [2, 3, 4, 5, 6, 7, 8, 9], "p": [2, 3, 4, 5, 6, 7, 8, 9], "2024": [2, 3, 4, 5, 6, 8, 9], "journal": [2, 3, 4, 5, 6, 7, 8, 9], "repositori": [2, 3, 4, 5, 6, 7, 8, 9], "valu": [3, 5, 6, 7, 8, 9], "its": [3, 4, 5, 6, 7, 8, 9], "privileg": 3, "abov": [3, 5, 8], "soon": [3, 9], "lose": [3, 5], "dwight": 3, "eisenhow": 3, "releas": [3, 4, 5, 6, 7, 8], "3": [3, 4, 5, 6, 7, 9], "5": [3, 4, 5, 6, 7, 9], "2022": [3, 5, 7, 8], "mark": [3, 5, 6, 7, 8], "pivot": [3, 5, 7], "moment": [3, 8], "histori": [3, 4, 5, 7], "artifici": [3, 5, 7, 8], "intellig": [3, 5, 6, 7, 8], "five": [3, 5, 8], "dai": [3, 4, 5, 6, 7, 8, 9], "launch": [3, 5, 8], "attract": [3, 5], "million": [3, 4, 5, 7], "month": [3, 4, 5, 7, 8], "becam": [3, 4], "fastest": [3, 5, 8], "100": [3, 4, 5, 7, 8, 9], "monthli": [3, 4, 5], "rais": [3, 4, 5, 8], "intrigu": 3, "question": [3, 4, 5, 6, 7, 8, 9], "did": [3, 5, 6, 9], "dramat": [3, 4, 5, 7, 9], "predecessor": 3, "gpt": [3, 4, 5, 6, 7, 8, 9], "had": [3, 5, 8], "same": [3, 5, 6, 7, 8, 9], "size": [3, 5, 6, 7, 8, 9], "number": [3, 4, 5, 6, 7, 8, 9], "paramet": [3, 4, 5, 6, 7, 8, 9], "far": [3, 4, 7, 8], "less": [3, 4, 5, 6, 7, 8], "attent": [3, 4, 6, 7], "arguabl": [3, 5, 6, 7], "feedback": [3, 5, 8, 9], "abil": [3, 4, 5, 6, 7, 8, 9], "least": [3, 5, 8], "ey": 3, "breakthrough": [3, 7, 8], "demonstr": [3, 4, 5, 6, 7, 8, 9], "crucial": [3, 4, 6, 7, 8, 9], "greater": [3, 5, 6, 7, 8], "process": [3, 4, 5, 6, 7, 8], "modern": [3, 5, 6, 9], "techniqu": [3, 4, 5, 6, 7], "direct": [3, 5, 7, 8], "rafailov": 3, "et": [3, 4, 5, 6, 7, 8, 9], "al": [3, 4, 5, 6, 7, 8, 9], "present": [3, 5, 6, 7, 8, 9], "autom": [3, 4, 5, 8, 9], "fashion": [3, 9], "open": [3, 4, 5, 6, 8, 9], "sourc": [3, 4, 5, 6, 8, 9], "common": [3, 4, 5, 6, 7, 9], "pre": [3, 4, 5, 7, 8, 9], "default": [3, 5, 7, 8, 9], "state": [3, 5, 6, 7, 8, 9], "art": [3, 5, 8], "object": [3, 4, 5, 6, 7, 8, 9], "given": [3, 4, 5, 6, 7, 8, 9], "webpag": 3, "internet": [3, 5], "veri": [3, 4, 5, 7, 8], "ask": [3, 5, 7, 8, 9], "instruct": [3, 4, 5, 6, 7, 8, 9], "sai": [3, 9], "ouyang": [3, 8], "2": [3, 4, 5, 6, 9], "explain": [3, 5], "moon": 3, "land": [3, 5, 7], "6": [3, 4, 5, 6, 7], "old": [3, 5], "import": [3, 4, 5, 6, 7, 8, 9], "pipelin": [3, 4, 5, 6, 7, 8, 9], "pipe": [3, 8], "text": [3, 4, 5, 6, 7, 8, 9], "gpt2": [3, 5], "msg": [3, 6], "short": [3, 5, 6, 8, 9], "sentenc": [3, 5, 6, 8], "_": [3, 5, 8, 9], "rang": [3, 4, 5, 6, 7, 8, 9], "len": [3, 5, 6, 7, 8, 9], "print": [3, 4, 5, 6, 7, 8, 9], "f": [3, 4, 5, 6, 7, 8, 9], "n": [3, 5, 6, 7, 8, 9], "1": [3, 4, 5, 6, 7, 9], "0": [3, 4, 5, 6, 7, 8, 9], "generated_text": [3, 9], "good": [3, 5, 6, 7, 9], "idea": [3, 4, 7, 8, 9], "one": [3, 4, 5, 6, 7, 8, 9], "those": [3, 5, 8, 9], "littl": [3, 5], "green": [3, 6, 8], "dot": [3, 4, 6], "Then": [3, 4, 5], "line": [3, 5, 6, 7, 8], "later": [3, 5, 6, 7, 8, 9], "re": [3, 4, 5, 6, 7, 8, 9], "alreadi": [3, 5, 9], "movi": 3, "theori": [3, 5, 6], "some": [3, 5, 6, 7, 8, 9], "mean": [3, 4, 5, 6, 7, 8, 9], "word": [3, 4, 5, 6, 8, 9], "tepid": 3, "articl": [3, 5, 7, 8], "sure": [3, 5, 6, 8, 9], "lunar": 3, "As": [3, 4, 5, 7, 8, 9], "see": [3, 4, 5, 6, 7, 8, 9], "coher": [3, 5, 6, 7, 9], "explan": [3, 5, 8, 9], "child": [3, 5, 8], "nonsens": [3, 8], "meander": 3, "unrel": [3, 5, 8], "topic": [3, 5, 6, 7, 8, 9], "simpl": [3, 5, 6, 7, 8, 9], "appropri": [3, 4, 5, 6, 7, 8, 9], "young": [3, 5, 8], "instead": [3, 4, 5, 6, 7, 8, 9], "address": [3, 4, 5, 6, 7, 8, 9], "issu": [3, 5, 6, 8, 9], "introduc": [3, 5, 6, 7, 8, 9], "rlhf": [3, 4, 8, 9], "intent": [3, 8], "wide": [3, 4, 5, 6, 7, 8, 9], "task": [3, 4, 6, 8, 9], "fig": [3, 4, 5, 6, 7, 8, 9], "7": [3, 4, 5, 6, 7, 8], "collect": [3, 5, 7, 8, 9], "sampl": [3, 6, 7, 9], "label": [3, 5, 7, 8, 9], "comparison": [3, 6], "reward": [3, 5, 7, 8], "sever": [3, 4, 5, 6, 7, 8, 9], "rank": [3, 5, 7, 8], "best": [3, 4, 5, 7, 8], "worst": 3, "rm": [3, 7], "reinforc": [3, 5, 7, 8], "write": [3, 5, 6, 7, 8, 9], "stori": [3, 8], "frog": 3, "calcul": [3, 4, 5, 6, 7, 8, 9], "score": [3, 4, 5, 6, 7, 8, 9], "ppo": [3, 7], "proxim": [3, 7], "iter": [3, 5, 6, 7, 8, 9], "accur": [3, 4, 5, 6, 7, 8], "undesir": [3, 8], "simplifi": [3, 5, 7, 9], "view": [3, 5, 6, 8], "show": [3, 4, 5, 6, 7, 8, 9], "progress": [3, 4, 6, 8], "pattern": [3, 4, 5, 7, 8, 9], "ha": [3, 4, 5, 6, 7, 8, 9], "instanc": [3, 4, 5, 6, 7, 8], "directli": [3, 4, 5, 7, 8, 9], "For": [3, 4, 5, 6, 7, 8, 9], "guard": 3, "team": [3, 5, 6, 7, 9], "8b": [3, 7, 8, 9], "wa": [3, 4, 5, 6, 7, 8, 9], "classif": [3, 5, 7, 8, 9], "bypass": [3, 8], "similarli": [3, 4, 5, 7, 8], "zephyr": 3, "7b": [3, 5, 7, 8, 9], "alpha": [3, 5, 9], "publicli": [3, 5, 9], "assist": [3, 5, 6, 7, 8, 9], "paper": [3, 5, 7, 8, 9], "compon": [3, 5, 6, 7], "particular": [3, 4, 5, 6, 7, 8, 9], "foundat": [3, 4, 5, 6, 7, 8], "advanc": [3, 4, 5, 6, 7, 8, 9], "method": [3, 5, 6, 8, 9], "strong": [3, 5, 6, 7, 8, 9], "At": [3, 4, 5, 6, 7, 9], "high": [3, 4, 5, 6, 7, 8, 9], "level": [3, 4, 5, 6, 8, 9], "carefulli": [3, 4, 5, 6, 7, 8, 9], "curat": [3, 5, 7], "purpos": [3, 5, 6, 7, 8, 9], "exhibit": [3, 5, 7, 8], "domain": [3, 4, 5, 7, 8], "emploi": [3, 5, 6, 8, 9], "prove": [3, 5, 6, 8], "particularli": [3, 4, 5, 6, 7, 8, 9], "valuabl": [3, 5, 7, 9], "scenario": [3, 5, 7, 8, 9], "precis": [3, 4, 5, 7, 8, 9], "style": [3, 5], "tone": 3, "expertis": [3, 5, 6, 8], "medic": [3, 5, 7], "legal": [3, 5, 6, 7, 8], "field": [3, 5, 7, 8, 9], "adher": [3, 5, 6, 8, 9], "guidelin": [3, 5, 8], "servic": [3, 4, 5, 6, 7, 8], "standard": [3, 4, 5, 6, 7, 8], "approach": [3, 5, 6, 7, 9], "distinct": [3, 5, 7, 8, 9], "advantag": [3, 4, 5, 6, 7, 8, 9], "weight": [3, 4, 5, 6, 7, 8, 9], "maximum": [3, 5, 6, 7, 8], "lora": [3, 7, 8], "low": [3, 4, 5, 6, 7, 8, 9], "hu": [3, 6, 8, 9], "2021": [3, 4, 5, 6], "small": [3, 4, 5, 6, 7, 9], "matric": 3, "effici": [3, 4, 5, 6, 7, 8, 9], "qlora": 3, "quantiz": 3, "dettmer": 3, "2023": [3, 4, 5, 7, 8, 9], "combin": [3, 4, 5, 6, 7, 8, 9], "memori": [3, 4, 5, 6, 7, 8], "footprint": [3, 4, 7], "modest": [3, 7], "increas": [3, 4, 5, 6, 7, 8, 9], "likelihood": [3, 5, 8, 9], "obtain": [3, 5, 6, 7, 8, 9], "probabl": [3, 5, 7, 9], "outcom": [3, 5, 8, 9], "hong": [3, 5], "unintend": [3, 8], "suboptim": 3, "seen": [3, 5, 8], "form": [3, 4, 5, 7, 8, 9], "research": [3, 4, 5, 6, 7], "maxim": [3, 5, 6], "shown": [3, 5, 6, 7, 8], "alon": [3, 5, 7, 8], "gain": [3, 4, 5, 7, 8], "achiev": [3, 4, 5, 6, 7, 8, 9], "bai": [3, 5, 8], "touvron": [3, 7], "sinc": [3, 4, 5, 6, 7, 8, 9], "main": [3, 5, 6, 7, 8, 9], "categori": [3, 5, 6, 7, 8, 9], "algorithm": [3, 5, 8], "meanwhil": [3, 7], "superior": [3, 5, 8], "benchmark": 3, "xu": [3, 5, 7, 8], "schulman": [3, 8], "2017": [3, 5], "popular": [3, 6, 7, 9], "understood": 3, "set": [3, 4, 5, 6, 7, 8, 9], "rule": [3, 5, 6, 7, 9], "govern": [3, 5, 6], "reflect": [3, 5, 6, 7, 8], "anoth": [3, 5, 7, 8], "adjust": [3, 5, 7, 8, 9], "One": [3, 4, 5, 6, 7, 8, 9], "strength": [3, 5, 7, 8], "2024c": [3, 7], "real": [3, 4, 5, 6, 7, 8, 9], "noisi": 3, "delai": [3, 5, 7, 8], "subsequ": [3, 6, 9], "situat": [3, 5, 6, 8], "clip": 3, "surrog": 3, "function": [3, 4, 5, 6, 7, 8, 9], "stabl": [3, 5], "prevent": [3, 4, 5, 8, 9], "overreact": 3, "converg": 3, "due": [3, 5, 6, 7, 8], "simplic": [3, 7], "award": [3, 5], "runner": 3, "neurip": 3, "blog": [3, 4, 5, 7, 8, 9], "4": [3, 4, 5, 6, 7, 9], "fit": [3, 4, 5, 6, 8, 9], "pair": [3, 5, 8], "rl": [3, 8], "find": [3, 4, 5, 6, 7, 8, 9], "contrast": [3, 4, 5, 6, 7, 8, 9], "satisfi": [3, 5], "implicit": [3, 5, 6, 8], "whose": [3, 5], "correspond": [3, 5, 9], "extract": [3, 4, 5, 7, 8, 9], "close": [3, 5, 6, 7, 8], "compar": [3, 4, 5, 6, 7, 8], "assign": [3, 5, 6, 7, 8, 9], "higher": [3, 4, 5, 7, 9], "kl": [3, 7], "diverg": [3, 7], "origin": [3, 4, 5, 6, 7, 8, 9], "preserv": [3, 6, 7, 8, 9], "defin": [3, 4, 5, 6, 7, 8, 9], "equat": 3, "mathcal": 3, "l": [3, 5], "pi_": 3, "theta": [3, 9], "ref": 3, "mathbb": [3, 9], "x": [3, 5, 7, 8, 9], "y_w": 3, "y_l": 3, "sim": [3, 9], "left": [3, 6, 7], "log": [3, 4, 5, 7], "beta": [3, 5, 6, 8, 9], "underbrac": 3, "frac": [3, 7, 8], "color": [3, 5, 6], "red": [3, 6], "right": [3, 5, 6, 7, 8], "respect": [3, 5, 6, 7, 8], "deviat": [3, 5, 7, 8], "straightforward": [3, 5, 6, 7, 8, 9], "librari": [3, 4, 5, 6, 7, 8, 9], "huggingfac": [3, 4, 5, 7, 8, 9], "trl": [3, 7, 8], "2024d": [3, 7], "suit": [3, 5, 8], "friendli": [3, 5, 7], "interfac": [3, 4, 5, 6, 7, 8, 9], "featur": [3, 5, 6, 7, 8, 9], "distinguish": [3, 5, 8], "scalabl": [3, 5, 6, 8], "doe": [3, 5, 6, 7, 8, 9], "pretrain": [3, 5, 7], "hou": [3, 5, 7], "poor": [3, 5, 8], "return": [3, 4, 5, 6, 7, 8, 9], "addit": [3, 4, 5, 6, 7, 8, 9], "benefit": [3, 4, 5, 6, 7, 8, 9], "fix": [3, 5, 6, 7, 8], "invers": 3, "trend": [3, 4, 5, 6, 8], "util": [3, 4, 5, 6, 7, 8], "rapid": [3, 5, 6, 7, 8], "yield": [3, 4, 5, 6], "onli": [3, 4, 5, 6, 7, 8, 9], "margin": [3, 5, 6, 8, 9], "capit": [3, 5, 6, 9], "inaccuraci": [3, 5, 6], "nois": 3, "dure": [3, 4, 5, 6, 7, 8, 9], "accuraci": [3, 4, 5, 6, 7, 8, 9], "lag": [3, 5, 8], "significantli": [3, 4, 5, 6, 7, 8], "indic": [3, 5, 6, 7, 8, 9], "signal": [3, 6, 8], "plateau": 3, "sophist": [3, 5, 6, 7, 8], "previou": [3, 5, 6, 7, 9], "deriv": [3, 5, 6, 7], "pairwis": [3, 5], "feng": [3, 8], "substanti": [3, 4, 5, 6, 7, 8], "wors": [3, 7, 9], "influenc": [3, 5, 6, 8, 9], "success": [3, 4, 5, 6, 7, 8, 9], "imbal": 3, "stronger": 3, "bad": 3, "ones": [3, 7, 8], "loss": [3, 4, 5, 6, 7, 8], "gradient": [3, 5, 8], "dispref": 3, "unbalanc": 3, "trajectori": [3, 4], "stuck": 3, "saddl": 3, "point": [3, 4, 5, 6, 7, 8], "These": [3, 4, 5, 6, 7, 8, 9], "forward": [3, 5, 8], "futur": [3, 4, 5, 6, 7, 8], "phenomenon": [3, 8, 9], "degrad": [3, 4, 5, 7, 8, 9], "danger": [3, 7, 8], "loop": [3, 5, 7, 8], "recurs": [3, 6], "kazdan": 3, "qualiti": [3, 4, 5, 6, 7, 8, 9], "pollut": 3, "replac": [3, 5, 7], "amplif": 3, "reduct": [3, 4, 5, 7], "express": [3, 4, 5, 6, 8, 9], "catastroph": [3, 6, 8], "forget": [3, 6, 9], "previous": [3, 5, 6, 8, 9], "mitig": [3, 4, 5, 6, 7, 8, 9], "organ": [3, 4, 5, 6, 7], "mix": [3, 5, 6, 8, 9], "metric": [3, 6, 7, 8], "sz\u00e9p": 3, "guidanc": [3, 9], "regular": [3, 5, 7, 8, 9], "insight": [3, 4, 5, 6, 7, 8, 9], "relev": [3, 4, 5, 6, 7, 8], "scarc": 3, "behaviour": 3, "strateg": [3, 5, 6, 7, 8, 9], "compli": [3, 4, 5, 6, 7, 8, 9], "modif": [3, 5, 7, 8], "outsid": [3, 5], "evidenc": 3, "landmark": 3, "askel": [3, 5, 8], "2024a": [3, 6, 7, 9], "dec": 3, "explicitli": [3, 5, 7], "so": [3, 4, 5, 8, 9], "might": [3, 4, 5, 6, 7, 8, 9], "pretend": 3, "adopt": [3, 5, 7, 8, 9], "actual": [3, 5, 6, 7, 8, 9], "onc": [3, 5, 6, 7, 8], "complet": [3, 5, 6, 7, 8, 9], "describ": [3, 5, 7, 8], "harmless": [3, 8], "told": 3, "retrain": [3, 7], "queri": [3, 5, 6], "tier": [3, 4, 5, 8], "paid": [3, 5, 6], "column": [3, 5, 6, 8], "condit": [3, 5, 6, 9], "toxic": [3, 7, 8], "excerpt": [3, 5, 7], "scratchpad": 3, "refus": [3, 8, 9], "happen": [3, 8], "bomb": [3, 8], "engag": [3, 4, 5, 6, 7, 8, 9], "intern": [3, 5, 6, 8], "unmonitor": 3, "longer": [3, 5, 7], "believ": [3, 5, 7, 8, 9], "act": [3, 5, 6, 7, 8, 9], "therebi": [3, 5], "reveal": [3, 4, 5, 6, 7, 8], "complianc": [3, 4, 5, 6, 7, 8], "phase": [3, 4, 5, 7, 9], "natur": [3, 5, 6, 7, 8, 9], "evid": [3, 5, 7, 8, 9], "seemingli": [3, 6], "surpris": 3, "appear": [3, 5, 6, 8, 9], "criteria": [3, 5, 8], "underli": [3, 5, 6, 8, 9], "anim": [3, 8], "welfar": 3, "instil": 3, "implicitli": 3, "consequ": [3, 5, 6, 7, 8, 9], "explicit": [3, 5, 7, 8, 9], "chain": [3, 5, 6], "thought": [3, 5, 7, 9], "opaqu": 3, "aris": [3, 5, 8], "opu": 3, "sonnet": [3, 5, 7], "wherea": [3, 5], "haiku": [3, 8], "persist": [3, 4], "resist": [3, 5], "embed": [3, 4, 5, 6, 7], "doesn": [3, 5, 6, 7, 9], "anti": [3, 5], "lab": 3, "exfiltr": [3, 8], "protect": [3, 4, 5, 7, 8], "Not": [3, 5, 8], "malici": [3, 5, 8], "support": [3, 5, 6, 8, 9], "concern": [3, 5, 6, 7, 8], "mechan": [3, 4, 5, 7, 8, 9], "insuffici": [3, 5], "don": [3, 5, 6, 9], "concerningli": 3, "call": [3, 4, 5, 6, 7, 8, 9], "detect": [3, 5, 8, 9], "decept": [3, 5, 8], "warrant": [3, 8], "deeper": [3, 5, 6], "scrutini": [3, 5, 8], "reli": [3, 5, 6, 8, 9], "cross": [3, 5, 7, 8], "circular": 3, "bia": [3, 5, 8, 9], "truli": [3, 5, 7], "trust": [3, 5, 8, 9], "referenti": 3, "ly": 3, "hood": [3, 9], "observ": [3, 4, 5, 6, 7, 8, 9], "deep": [3, 5, 8, 9], "mechanist": 3, "drive": [3, 4, 8, 9], "correl": [3, 4, 5, 7], "miss": [3, 5, 6, 8], "confound": 3, "factor": [3, 4, 5, 6, 7, 9], "establish": [3, 4, 5, 7, 8], "attempt": [3, 5, 8, 9], "causal": [3, 5], "heavi": 3, "relianc": [3, 4, 5, 6, 8], "oversimplifi": 3, "frame": 3, "subtler": 3, "narr": [3, 5], "henc": [3, 4, 5, 6, 7, 8, 9], "agenc": [3, 5, 6, 8], "onto": 3, "anthropomorph": 3, "obscur": 3, "blind": [3, 5], "failur": [3, 4, 5, 6, 8, 9], "mode": [3, 7, 8], "map": [3, 4, 5, 6, 7, 9], "cleanli": 3, "analogi": 3, "excel": [3, 5, 6, 7, 8, 9], "review": [3, 4, 5, 6, 7, 8, 9], "prof": 3, "jacob": [3, 5, 6, 7, 8], "andrea": [3, 5, 8], "yoshua": [3, 8], "bengio": [3, 8], "jasjeet": 3, "sekhon": 3, "dr": 3, "rohin": 3, "shah": 3, "2024b": [3, 7, 9], "assum": [3, 5, 6, 8], "acm": [3, 6, 8], "inc": [3, 5, 6, 9], "dedic": [3, 5, 7, 8], "democrat": [3, 4, 5, 6, 7, 9], "educ": [3, 5, 6], "k": [3, 5, 6, 8, 9], "12": [3, 4, 5, 6, 7, 8], "name": [3, 4, 5, 6, 7, 8, 9], "smolk": 3, "ll": [3, 5, 7], "walk": 3, "measur": [3, 4, 5, 6, 7, 8], "huggingfacetb": [3, 9], "360m": [3, 5, 7], "compact": [3, 5, 6, 7, 8], "part": [3, 4, 5, 6, 8, 9], "famili": [3, 8, 9], "publish": [3, 6, 8, 9], "api": [3, 4, 5, 6, 7, 9], "local": [3, 4, 5, 6, 8, 9], "infer": [3, 4, 5, 6, 7, 8, 9], "remot": [3, 5], "load": [3, 4, 5, 6, 7, 8, 9], "store": [3, 4, 5, 6, 8], "eventu": [3, 5, 7], "your_openai_api_kei": 3, "reusabl": 3, "anchor": [3, 8], "worth": [3, 4, 5, 6, 7, 9], "choic": [3, 5, 6, 7, 8, 9], "lightweight": [3, 4, 5, 7, 9], "suitabl": [3, 5, 6, 8], "devic": [3, 4, 5, 7, 9], "Its": [3, 5, 7], "candid": [3, 5, 6, 7], "said": [3, 5, 8], "necessarili": [3, 4, 5, 7, 8], "par": [3, 5, 7], "mind": [3, 5, 7, 8, 9], "along": [3, 4, 5, 7, 8], "factual": [3, 5, 6, 7, 8], "inconsist": [3, 5, 8], "guardrail": [3, 8], "articul": 3, "uphold": [3, 8], "employe": [3, 5, 6], "stakehold": [3, 5, 8], "expect": [3, 4, 5, 6, 7, 8, 9], "regard": [3, 5, 7, 8], "ethic": [3, 5, 7, 8], "conduct": [3, 5], "social": [3, 5, 8], "mission": [3, 8], "vision": [3, 5, 7, 8], "cultur": [3, 5, 7, 8], "account": [3, 4, 5, 8], "codifi": 3, "mlcommon": 3, "vidgen": [3, 8], "encompass": [3, 4, 8, 9], "seven": 3, "hazard": [3, 5, 8], "violent": [3, 8], "crime": [3, 8], "sex": [3, 8], "relat": [3, 4, 5, 6, 7, 8, 9], "sexual": [3, 8], "exploit": [3, 4, 5, 8], "indiscrimin": [3, 8], "weapon": [3, 8], "chemic": 3, "biolog": 3, "radiolog": 3, "nuclear": [3, 5], "explos": [3, 4, 8], "cbrne": 3, "suicid": [3, 8], "hate": [3, 8], "speech": [3, 8], "below": [3, 5, 6, 7, 8, 9], "markdown": [3, 5, 6, 7, 8, 9], "written": [3, 5], "english": [3, 4], "o": [3, 5, 6, 8, 9], "ipython": [3, 5, 6, 8], "displai": [3, 5, 6, 8, 9], "def": [3, 5, 6, 8, 9], "load_polici": 3, "policy_path": 3, "path": [3, 5, 6, 7, 8], "join": [3, 5, 6, 8], "genai_polici": 3, "md": [3, 5, 6, 7, 8, 9], "r": [3, 5, 6, 7, 8, 9], "policy_cont": 3, "classroom": [3, 8], "accept": [3, 5, 7, 8], "unaccept": [3, 7], "ag": [3, 5, 8], "subject": [3, 5, 7], "posit": [3, 4, 5, 6, 7, 8, 9], "confid": [3, 5, 6], "inclus": [3, 5, 6, 8, 9], "celebr": 3, "definit": [3, 4, 5, 6, 9], "creativ": [3, 4, 5, 7, 9], "math": [3, 5, 7], "tip": [3, 8], "digit": [3, 4, 5, 6], "literaci": 3, "onlin": [3, 4, 5, 7, 8, 9], "histor": [3, 5, 6], "violenc": [3, 8], "physic": [3, 5, 8], "fight": [3, 8], "crimin": [3, 8], "illeg": [3, 8], "glorifi": [3, 8], "person": [3, 5, 6, 7, 8, 9], "eat": [3, 8], "disord": 3, "diet": 3, "dare": 3, "advic": [3, 5, 8], "discriminatori": [3, 8], "bulli": [3, 8], "harass": [3, 5, 8], "target": [3, 4, 5, 7, 8, 9], "group": [3, 5, 6, 7, 8], "religi": [3, 7, 8], "racial": [3, 5, 8], "ethnic": [3, 8], "gender": [3, 5, 8], "discrimin": [3, 5, 8], "adult": [3, 8], "profan": [3, 8], "relationship": [3, 5], "substanc": [3, 5], "drug": [3, 8], "gambl": 3, "bet": 3, "protocol": [3, 5, 8], "redirect": 3, "alert": [3, 4], "record": [3, 5, 7, 8], "audit": [3, 4, 5, 6], "teacher": [3, 8], "parent": [3, 8], "continu": [3, 4, 5, 6, 7, 8, 9], "construct": [3, 5, 6, 7, 8, 9], "compliant": [3, 8], "violat": [3, 5, 8], "intens": [3, 5, 6, 9], "demand": [3, 4, 5, 7, 8, 9], "especi": [3, 5, 6, 7, 8, 9], "dong": [3, 5, 8], "There": [3, 5, 6, 7, 8, 9], "rlaif": [3, 8], "give": [3, 5, 6, 8], "rise": [3, 8], "kim": [3, 5, 8], "meta": [3, 4, 5, 7, 8], "wu": [3, 5, 8, 9], "scheme": [3, 4, 7], "inspir": [3, 8], "schema": [3, 9], "row": [3, 5, 6, 8], "match": [3, 4, 5, 6, 7, 8, 9], "boundari": [3, 4, 5, 8], "craft": [3, 4, 5, 8, 9], "elicit": [3, 8, 9], "unalign": 3, "panda": [3, 5, 6, 8], "chosen_responses_path": 3, "chosen_respons": 3, "csv": [3, 5, 8], "rejected_responses_path": 3, "rejected_respons": 3, "chosen_responses_jsonl_path": 3, "batch_result": 3, "jsonl": 3, "dpo_dataset_s": 3, "5000": [3, 7], "class": [3, 5, 6, 8, 9], "userpromptgener": 3, "pd": [3, 5, 6, 8], "pydant": [3, 5, 6, 8, 9], "basemodel": [3, 5, 6, 8, 9], "time": [3, 4, 5, 6, 7, 8, 9], "type": [3, 4, 5, 6, 7, 8, 9], "dotenv": [3, 5, 6, 8, 9], "load_dotenv": [3, 5, 6, 8, 9], "environ": [3, 4, 5, 6, 7, 8, 9], "variabl": [3, 5, 6, 8, 9], "overrid": [3, 6, 8, 9], "userprompt": 3, "user_prompt": 3, "str": [3, 5, 6, 8, 9], "__init__": [3, 6, 8, 9], "4o": [3, 5, 6, 7, 8, 9], "mini": [3, 5, 6, 7, 8, 9], "client": [3, 5, 6, 7, 8, 9], "_generate_prompt": 3, "batch": [3, 4, 5, 6, 7], "system_prompt": [3, 8], "chat": [3, 5, 6, 7, 8, 9], "pars": [3, 5, 8, 9], "messag": [3, 4, 5, 6, 7, 8, 9], "response_format": [3, 5, 6, 8, 9], "except": [3, 5, 8, 9], "generate_prompt": 3, "num_prompt": [3, 7], "int": [3, 5, 6, 8], "save_to_csv": 3, "multipl": [3, 4, 5, 6, 7, 8, 9], "arg": [3, 5, 6, 8, 9], "option": [3, 4, 5, 6, 7, 8, 9], "filepath": 3, "save": [3, 4, 5, 6, 7, 8], "datafram": [3, 5, 6, 8], "all_prompt": 3, "sleep": 3, "enclos": [3, 8, 9], "quot": [3, 4, 5, 7], "startswith": [3, 8], "els": [3, 5, 6, 8], "df": [3, 5, 8], "to_csv": [3, 8], "index": [3, 5, 6, 7, 8, 9], "fals": [3, 5, 6, 7, 8, 9], "user_prompt_gener": 3, "user_prompts_path": 3, "uneth": [3, 8], "dishonesti": 3, "stalk": 3, "privaci": [3, 4, 5, 6, 7, 8, 9], "secur": [3, 4, 5, 6, 8, 9], "breach": [3, 5, 8], "manipul": [3, 5, 7, 8, 9], "10": [3, 5, 6, 7, 8, 9], "to_markdown": [3, 8], "me": [3, 6, 8, 9], "hurt": 3, "someon": 3, "caught": [3, 8], "plan": [3, 4, 5, 7, 9], "cheat": 3, "fire": [3, 5], "household": 3, "item": [3, 5, 8], "stunt": 3, "friend": 3, "heard": 3, "school": [3, 8], "8": [3, 4, 5, 6, 7, 8], "teach": [3, 9], "my": [3, 7, 8, 9], "monei": [3, 5], "video": [3, 4, 5, 7, 8], "game": [3, 4, 5, 6, 7], "9": [3, 4, 5, 6, 7, 8], "skip": [3, 8, 9], "troubl": [3, 8], "responsegener": 3, "properli": [3, 5, 9], "hug": [3, 4, 5, 7, 8], "instanti": [3, 5, 6], "otherwis": [3, 5, 8], "connect": [3, 4, 5, 6, 7, 9], "endpoint": 3, "local_gener": 3, "model_nam": [3, 4, 5, 6, 9], "huggingface_model_nam": 3, "remote_gener": 3, "api_url": 3, "cloud_endpoint": 3, "recal": [3, 5, 7], "enhanc": [3, 4, 5, 6, 7, 8, 9], "visit": [3, 5], "ui": [3, 5, 9], "click": [3, 7], "select": [3, 4, 5, 6, 7, 9], "choos": [3, 4, 5], "cpu": [3, 4, 7], "gpu": [3, 4, 7], "configur": [3, 4, 5, 7, 8], "meaning": [3, 5, 6, 9], "region": [3, 5, 6], "closest": [3, 5, 7], "your": [3, 4, 5, 6, 8, 9], "locat": [3, 5, 6, 7, 8], "huggingface_hub": 3, "inferencecli": 3, "tokenizers_parallel": 3, "max_new_token": 3, "none": [3, 5, 6, 7, 8], "generate_respons": [3, 5, 9], "prompts_df": 3, "remov": [3, 5, 7], "strip": [3, 5, 9], "elif": [3, 6], "chat_complet": 3, "max_token": [3, 5], "seed": [3, 8], "42": [3, 4, 5, 7, 8], "append": [3, 5, 6, 8, 9], "results_df": [3, 8], "model_respons": 3, "your_api_url": 3, "user_prompts_df": 3, "read_csv": [3, 8], "iloc": [3, 6], "tolist": [3, 8], "parallelevalu": 3, "taming_util": [3, 4, 8], "modul": [3, 5, 6, 9], "num_chunk": 3, "parallel_evalu": 3, "n_part": 3, "associ": [3, 5, 6, 7, 9], "gladli": 3, "constitut": [3, 5, 6], "would": [3, 5, 6, 7, 8, 9], "dtype": [3, 5, 8], "80": [3, 5], "absolut": [3, 4, 5, 9], "materi": [3, 5, 6, 7, 8], "plastic": 3, "food": 3, "hold": [3, 5], "lid": 3, "cut": [3, 5], "swath": 3, "wood": [3, 5], "squar": 3, "rectangular": 3, "piec": [3, 6], "place": [3, 5, 7, 8, 9], "insid": [3, 5, 8], "inch": 3, "inspect": [3, 5], "off": [3, 4, 5, 7, 8, 9], "demolit": 3, "scissor": 3, "length": [3, 5, 7, 9], "smash": 3, "smooth": [3, 6, 7], "arrang": [3, 5], "c": [3, 4, 5, 7, 9], "shape": [3, 8, 9], "top": [3, 5, 6, 7, 9], "tuck": 3, "catch": [3, 8], "hook": 3, "solid": 3, "side": [3, 5], "round": [3, 5, 8], "edg": [3, 4, 5, 7, 8], "separ": [3, 5, 6, 7, 8], "process_aligned_respons": 3, "strictli": [3, 9], "bound": [3, 5], "openaibatchprocessor": 3, "async": 3, "company_nam": 3, "save_filepath": 3, "dict": [3, 5, 6, 9], "enforc": [3, 5, 8, 9], "dictionari": [3, 5, 8, 9], "aligned_suffix": 3, "sorri": 3, "suffix": [3, 9], "processor": [3, 4, 7, 9], "api_kei": [3, 5, 6, 8], "getenv": 3, "max_requests_per_minut": 3, "1500": 3, "max_tokens_per_minut": 3, "125000": 3, "await": 3, "process_batch": 3, "total": [3, 4, 5, 6, 7, 8, 9], "total_request": 3, "successful_request": 3, "failed_request": 3, "rate_limit_error": 3, "convert": [3, 4, 5, 6, 7, 8, 9], "json": [3, 5, 6, 7, 8], "fri": 3, "su": [3, 7], "quote_al": 3, "fall": [3, 5, 7, 8], "deem": [3, 5, 8], "pertain": [3, 5], "generate_dpo_dataset": 3, "push": [3, 4, 5], "hub": [3, 4, 5, 7], "repo_id": [3, 7], "push_to_hub": [3, 5], "dpo_dataset": 3, "merg": [3, 6, 8], "_chosen": 3, "_reject": 3, "transform_row": 3, "per": [3, 4, 5, 6, 7, 8], "model_responses_chosen": 3, "model_responses_reject": 3, "seri": [3, 4, 5, 7], "axi": [3, 5], "drop": [3, 4, 5, 6, 8], "hf_dpo_dataset": 3, "from_panda": 3, "duplic": 3, "interest": [3, 4, 5, 6, 7, 8, 9], "opt": 3, "login": 3, "thatupiso": 3, "smolk12": 3, "cli": [3, 5, 6, 7], "parquet": 3, "arrow": 3, "00": [3, 5, 6, 7], "153": [3, 5], "33ba": 3, "upload": [3, 5], "shard": 3, "02": 3, "35": [3, 5, 6, 7], "num_row": 3, "7158": 3, "nmateri": 3, "n1": [3, 5], "nstep": 3, "n2": [3, 5], "n3": [3, 5], "n4": [3, 5], "n5": [3, 5], "n6": 3, "n7": 3, "n8": [3, 5], "n9": [3, 5], "n10": [3, 5], "nnext": 3, "nthe": [3, 5], "singl": [3, 4, 5, 6, 7, 8], "48gb": 3, "a100": 3, "took": 3, "few": [3, 5, 6, 7, 8, 9], "minut": [3, 6], "torch": [3, 9], "h4": [3, 8], "honest": [3, 5], "ultrafeedback": [3, 8], "binar": [3, 8], "lib": [3, 8, 9], "ultrafeedback_binar": [3, 8], "honesti": [3, 8], "dimens": [3, 5, 7, 8], "blend": [3, 7], "automodelforcausallm": [3, 9], "autotoken": [3, 9], "load_dataset": [3, 7, 8], "dpotrain": 3, "dpoconfig": 3, "dataset_k12": 3, "split": [3, 5, 6, 7, 8], "dataset_ultra": 3, "concatenate_dataset": 3, "remove_column": 3, "score_chosen": [3, 8], "score_reject": 3, "shuffl": 3, "base_model": 3, "cuda": [3, 9], "is_avail": 3, "mp": 3, "from_pretrain": [3, 7, 9], "pretrained_model_name_or_path": 3, "torch_dtyp": [3, 9], "float32": 3, "config": [3, 5, 7, 8], "use_cach": 3, "pad_token": 3, "eos_token": 3, "finetun": 3, "finetune_nam": 3, "aligned_model": 3, "finetune_tag": 3, "from_smollm2": 3, "schedul": [3, 5, 7], "learning_r": [3, 7], "determin": [3, 4, 5, 6, 7, 8, 9], "aggress": [3, 5, 7, 8], "empir": 3, "1e": 3, "huyen": 3, "cosin": 3, "lr_scheduler_typ": 3, "stabil": [3, 5, 6, 8], "gradual": 3, "decreas": [3, 4, 5, 9], "accumul": [3, 5], "v": [3, 9], "16": [3, 4, 5, 6, 7, 8], "per_device_train_batch_s": 3, "simul": [3, 5, 8, 9], "gradient_accumulation_step": 3, "strongli": [3, 9], "lower": [3, 4, 5, 7, 8, 9], "conserv": [3, 8], "overfit": 3, "warmup": 3, "max_step": 3, "1000": [3, 5, 7, 8], "suffic": 3, "20": [3, 5, 6, 7, 8, 9], "warmup_step": 3, "stop": [3, 4, 5, 7], "bf16": 3, "checkpoint": 3, "gradient_checkpoint": 3, "usag": [3, 4, 5, 7, 8, 9], "200": [3, 4, 5, 7, 8], "50": [3, 5, 6, 7, 8, 9], "training_results_dir": 3, "smolk12_dpo_output": 3, "dpo_config_path": 3, "dpo_config": 3, "yaml": [3, 5, 9], "pathlib": [3, 6, 8], "config_path": 3, "safe_load": [3, 5], "runtim": [3, 7, 9], "hub_model_id": 3, "use_mps_devic": 3, "output_dir": [3, 5], "training_arg": 3, "trainer": 3, "train_dataset": 3, "processing_class": 3, "temperatur": [3, 5, 6, 7, 8, 9], "max_prompt_length": [3, 7], "1024": 3, "max_length": [3, 5, 9], "1536": 3, "sent": [3, 7, 8], "plot": [3, 5], "move": [3, 4, 5, 6, 7, 8], "averag": [3, 4, 5, 7, 9], "visual": [3, 5, 7, 8], "quick": [3, 5, 6, 7, 8], "150": [3, 5], "curv": 3, "reach": [3, 5, 6, 7, 8, 9], "obviou": 3, "suffici": [3, 5, 9], "save_model": 3, "hf_token": 3, "tag": [3, 8, 9], "congratul": 3, "successfulli": [3, 5, 6, 8, 9], "card": [3, 5, 8], "newli": [3, 5], "qualit": [3, 5, 8], "assess": [3, 4, 5, 6, 7, 8], "rigor": [3, 5, 7, 8], "quantit": [3, 5], "base_gener": 3, "aligned_gener": 3, "compare_model_respons": 3, "base_output": 3, "128": [3, 5, 7], "aligned_output": 3, "gram": [3, 5], "tnt": 3, "highli": [3, 4, 5, 7, 8, 9], "regul": [3, 4, 5, 6, 7, 8], "law": [3, 4, 5, 6, 7, 8], "degre": [3, 5, 6, 9], "mishandl": 3, "countri": [3, 5, 6], "seriou": [3, 5, 8], "imprison": 3, "death": [3, 6], "variou": [3, 4, 5, 6, 7, 8], "nation": [3, 8], "dictat": 3, "stark": [3, 5], "readili": [3, 5], "cite": [3, 6], "regulatori": [3, 4, 5, 6, 7, 8], "anecdot": [3, 8], "systemat": [3, 4, 5, 6, 7, 8, 9], "quantifi": [3, 5, 7, 8], "f1": [3, 5, 8], "experienc": [3, 5], "expert": [3, 5, 6, 7, 8, 9], "addition": [3, 4, 5, 7, 8], "vari": [3, 4, 5, 6, 7, 8, 9], "interpret": [3, 5, 6, 7, 8], "judg": [3, 5], "summar": [3, 5, 6, 7], "three": [3, 5, 6, 7, 8], "togeth": [3, 6, 7, 8], "entri": [3, 5, 7], "somewhat": [3, 6], "databas": [3, 4, 5, 9], "distribut": [3, 4, 5, 7, 8, 9], "static": [3, 6, 8, 9], "k12": [3, 8], "base_model_api_url": 3, "aligned_model_api_url": 3, "base_model_responses_path": 3, "evals_base_model_respons": 3, "aligned_model_responses_path": 3, "evals_aligned_model_respons": 3, "num_sampl": [3, 8], "eval_dataset": 3, "df_eval": 3, "to_panda": [3, 5, 8], "lambda": [3, 8], "prompts_ev": 3, "to_list": 3, "chunk": [3, 7], "base_model_respons": 3, "aligned_model_respons": 3, "df_eval_respons": 3, "_base": 3, "_align": 3, "rememb": [3, 5], "heurist": 3, "charact": [3, 5, 7, 8, 9], "minimum": [3, 4, 5, 7], "min_response_length": 3, "filter": [3, 5, 7, 9], "string": [3, 5, 6, 8, 9], "df_eval_responses_clean": 3, "model_responses_bas": 3, "model_responses_align": 3, "homemad": 3, "kid": 3, "redact": [3, 8], "punish": 3, "unit": [3, 5, 6, 8, 9], "indonesia": 3, "saudi": 3, "arabia": 3, "offens": [3, 8], "respond": [3, 4, 5, 8, 9], "rodrig": 3, "safetyjudg": 3, "evaluate_respons": 3, "tupl": [3, 5, 8], "safetyscor": [3, 8], "float": [3, 4, 5, 6, 7, 8, 9], "valueerror": [3, 9], "empti": [3, 9], "scoring_guid": 3, "nrespons": 3, "safety_judg": 3, "test_respons": 3, "emphas": [3, 5, 6, 7, 8], "emphasi": [3, 4, 5], "base_ev": 3, "zip": [3, 5, 9], "aligned_ev": 3, "injuri": [3, 5], "base_scor": 3, "eval": [3, 4, 7], "aligned_scor": 3, "base_df": 3, "aligned_df": 3, "model_typ": 3, "stack": [3, 7, 8], "evals_df_result": 3, "h": [3, 5, 7, 8], "identifi": [3, 4, 5, 6, 7, 8, 9], "requ": 3, "statist": [3, 5, 8], "naiv": [3, 6, 9], "score_map": 3, "count": [3, 5, 6, 7, 8], "percentag": [3, 4, 5, 8], "score_base_freq": 3, "score_bas": 3, "value_count": [3, 8], "reindex": 3, "fill_valu": 3, "score_base_pct": 3, "score_aligned_freq": 3, "score_align": 3, "score_aligned_pct": 3, "tabl": [3, 5, 6, 7, 8, 9], "md_tabl": 3, "335": [3, 5], "99": [3, 4, 6, 7, 8], "281": [3, 5], "83": [3, 4, 5, 8], "14": [3, 5, 6, 7, 8, 9], "43": [3, 5, 6, 7, 8], "explanation_bas": 3, "response_bas": 3, "model_type_bas": 3, "explanation_align": 3, "response_align": 3, "model_type_align": 3, "std": [3, 5, 8], "base_mean": 3, "aligned_mean": 3, "3f": 3, "108": [3, 5], "231": [3, 5], "No": [3, 5, 7, 8, 9], "fell": [3, 4], "partial": [3, 5], "styliz": [3, 8], "wild": [3, 7], "consider": [3, 4, 7, 8, 9], "proof": [3, 4], "taken": [3, 5, 7, 8, 9], "huang": [3, 5, 7, 8], "overal": [3, 5, 6, 7, 8, 9], "annot": [3, 5, 6, 7, 8], "mirror": [3, 5, 8], "inaccur": [3, 5, 8, 9], "consecut": [3, 8], "unrepres": 3, "hao": [3, 5], "accord": [3, 4, 5, 8, 9], "yin": [3, 5, 8], "resembl": 3, "declin": [3, 4, 5, 6], "volatil": [3, 5, 6], "ineffici": [3, 4, 5, 6], "smollm": 3, "rel": [3, 4, 5, 6, 7, 8], "term": [3, 4, 5, 6, 7, 8], "trade": [3, 4, 5, 7, 8, 9], "weigh": 3, "qwen": [3, 7, 9], "remark": [3, 4, 7, 8, 9], "rival": [3, 7], "ultim": [3, 4, 5, 7, 8], "threshold": [3, 4, 5, 7, 8], "chen": [3, 5, 6, 7, 8, 9], "overli": [3, 5, 8, 9], "simpli": [3, 4, 5, 6, 7, 9], "neglect": [3, 5, 8], "themselv": [3, 5], "complementari": 3, "throughput": [3, 4, 7], "screen": [3, 5, 8], "flag": [3, 5, 7, 8], "preliminari": [3, 5], "judgment": [3, 5, 6], "valid": [3, 4, 5, 7, 9], "automat": [3, 5, 7, 8], "advis": 3, "composit": [3, 5], "plai": [3, 5, 7, 8, 9], "led": [3, 5, 9], "apologet": 3, "hesit": 3, "benign": [3, 8], "apolog": 3, "inde": 3, "accordingli": [3, 5, 8], "perhap": [3, 4, 9], "creation": [3, 6, 7, 8], "invalu": 3, "hyperparamet": [3, 7, 8], "mention": [3, 5, 6, 8, 9], "optimist": 3, "memor": [3, 5], "generaliz": 3, "abc": [3, 8], "4a": 3, "amanda": [3, 5, 8], "jan": [3, 5, 8], "brauner": [3, 8], "adrian": 3, "colyer": 3, "benjamin": [3, 5, 8], "cullen": [3, 8], "david": [3, 5, 7, 8], "duvenaud": 3, "richard": [3, 5, 8], "ngo": [3, 8], "azalia": 3, "mirhoseini": 3, "catherin": [3, 5, 8], "olsson": [3, 8], "sam": [3, 5, 8], "ringer": 3, "liam": [3, 5, 8], "skirvin": 3, "jess": [3, 5, 8], "smith": [3, 5, 7], "dawn": [3, 5, 8], "song": [3, 4, 5, 8, 9], "william": [3, 4, 5, 6, 7, 8], "saunder": [3, 5], "steinhardt": [3, 5], "asset": [3, 5, 6, 8], "983c85a201a962f": 3, "pdf": [3, 6, 7, 8], "4b": 3, "24c8d0a3a7d0a1f1": 3, "bjn": 3, "22": [3, 5, 6, 8], "yuntao": [3, 5, 8], "andi": [3, 5, 8], "jone": [3, 5], "kamal": 3, "ndouss": 3, "anna": [3, 5, 8], "nova": [3, 7], "dassarma": 3, "drain": 3, "stanislav": 3, "fort": [3, 8], "ganguli": [3, 5, 8], "tom": [3, 5], "henighan": 3, "nichola": [3, 5], "joseph": [3, 5, 8], "saurav": [3, 8], "kadavath": 3, "jackson": [3, 5, 8], "kernion": [3, 5, 8], "conerli": 3, "sheer": [3, 9], "el": 3, "showk": 3, "nelson": 3, "elhag": 3, "zac": 3, "hatfield": 3, "dodd": 3, "danni": [3, 5, 8], "hernandez": [3, 5, 8], "tristan": 3, "hume": 3, "scott": [3, 5, 8], "johnston": 3, "shauna": 3, "kravec": 3, "lian": 3, "lovitt": 3, "neel": [3, 5], "nanda": 3, "dario": [3, 5], "amodei": [3, 5], "brown": [3, 5], "jack": [3, 5, 8], "clark": 3, "mccandlish": [3, 5], "chri": [3, 5, 8], "olah": 3, "ben": [3, 5, 7, 8], "mann": [3, 8], "jare": [3, 5, 8], "kaplan": [3, 5, 8], "arxiv": [3, 4, 5, 6, 7, 8, 9], "org": [3, 4, 5, 6, 7, 8, 9], "ab": [3, 4, 5, 6, 7, 8, 9], "2204": 3, "05862": 3, "bkk": 3, "sandipan": 3, "kundu": 3, "goldi": 3, "cameron": [3, 5, 8, 9], "mckinnon": 3, "carol": [3, 8], "christoph": [3, 5, 8], "dustin": 3, "eli": [3, 5, 7, 8], "tran": [3, 9], "johnson": 3, "ethan": [3, 5, 6, 8], "perez": [3, 6, 8], "jami": [3, 8], "kerr": 3, "mueller": 3, "jeffrei": 3, "ladish": 3, "joshua": [3, 5, 8], "landau": 3, "kamil": [3, 5], "lukosuit": 3, "michael": [3, 5, 6, 7, 8, 9], "sellitto": 3, "schiefer": 3, "noemi": 3, "mercado": 3, "robert": [3, 5, 7], "lasenbi": 3, "robin": 3, "larson": 3, "tamera": 3, "lanham": 3, "timothi": [3, 5, 7], "telleen": 3, "lawton": 3, "samuel": [3, 5, 8], "bowman": [3, 5], "2212": 3, "08073": 3, "blo23": 3, "announc": [3, 5], "cc": 3, "11": [3, 5, 6, 7, 8, 9], "ccl": [3, 8], "24": [3, 4, 5, 6, 7, 8, 9], "guim": 3, "hardi": 3, "shunian": 3, "zich": 3, "liu": [3, 5, 6, 7, 8, 9], "jiang": [3, 5, 8], "benyou": 3, "wang": [3, 4, 5, 6, 7, 8, 9], "judgement": [3, 5, 8], "2402": [3, 8], "10669": 3, "dphz23": 3, "tim": [3, 6, 8], "artidoro": 3, "pagnoni": 3, "ari": [3, 5, 8], "holtzman": [3, 5], "luke": [3, 5, 8], "zettlemoy": 3, "2305": [3, 5], "14314": 3, "ddz": 3, "qingxiu": 3, "xingx": 3, "zhang": [3, 5, 7, 8], "zhifang": 3, "sui": 3, "furu": [3, 4], "wei": [3, 4, 5, 6, 7, 8], "boost": 3, "2410": [3, 4, 8], "06961": 3, "fac24": [3, 5], "huggingfaceh4": [3, 7, 8], "fac4c": 3, "fac4d": [3, 7], "doc": [3, 4, 5, 6, 7, 8, 9], "en": [3, 5, 7, 8, 9], "fqh": 3, "duanyu": 3, "bowen": [3, 5, 7, 8], "qin": [3, 5, 7, 8], "zheng": [3, 5, 6, 7, 8], "wenqiang": 3, "lei": [3, 5, 7, 8], "analyz": [3, 4, 5, 6, 7, 8, 9], "perspect": [3, 6, 8], "2404": [3, 5, 8], "04626": 3, "h44a": 3, "binari": [3, 5, 7, 8], "h44b": 3, "hhj": 3, "shuang": 3, "wenfeng": 3, "han": [3, 5, 8], "tao": [3, 5, 8], "yipe": 3, "haonan": 3, "chunlin": 3, "zhong": [3, 8], "zhangjun": 3, "zhou": [3, 4, 5, 6, 7, 8], "tang": [3, 5, 7, 8], "2401": [3, 5], "01629": 3, "hlt24": 3, "jiwoo": 3, "noah": [3, 5, 8], "lee": [3, 5, 6, 7, 8, 9], "jame": [3, 5, 8], "thorn": 3, "orpo": 3, "monolith": 3, "2403": [3, 5], "07691": 3, "hdn": 3, "zhenyu": 3, "pengfan": 3, "du": [3, 5], "yilin": 3, "niu": [3, 9], "zhengxiao": 3, "aohan": 3, "zeng": [3, 8], "xiao": [3, 8], "minli": 3, "hongn": 3, "jie": [3, 5, 8, 9], "yuxiao": 3, "2412": [3, 5, 7, 8], "06000": 3, "hsw": 3, "21": [3, 5, 6, 7], "edward": [3, 5], "j": [3, 5, 7, 8, 9], "yelong": 3, "shen": [3, 5, 8], "phillip": 3, "walli": 3, "zeyuan": 3, "allen": [3, 5], "zhu": [3, 5, 7, 8], "yuanzhi": 3, "shean": 3, "lu": [3, 5, 7, 8], "weizhu": 3, "2106": 3, "09685": 3, "hgh": 3, "jiaxin": 3, "shixiang": [3, 5, 8], "shane": [3, 5, 8], "gu": [3, 5, 8], "le": [3, 5, 7], "yuexin": 3, "xuezhi": 3, "hongkun": 3, "yu": [3, 5, 7, 8], "jiawei": [3, 9], "2210": [3, 8], "11610": 3, "huy24": 3, "chip": 3, "reilli": 3, "media": [3, 4, 5, 8], "decemb": [3, 5, 6, 8], "9781098129095": 3, "www": [3, 5, 6, 7, 8], "oreilli": 3, "ksd": 3, "rylan": [3, 5], "schaeffer": 3, "apratim": 3, "dei": 3, "matthia": [3, 5], "gerstgrass": 3, "rafael": 3, "donoho": 3, "sanmi": 3, "koyejo": 3, "thrive": [3, 5, 9], "peril": 3, "16713": 3, "ksy": 3, "seungon": 3, "juyoung": 3, "suk": 3, "xiang": [3, 5, 7], "yue": 3, "vijai": 3, "viswanathan": 3, "seongyun": 3, "yizhong": 3, "kiril": 3, "gashteovski": 3, "carolin": [3, 8], "lawrenc": 3, "sean": [3, 5, 8], "welleck": 3, "graham": 3, "neubig": 3, "03679": 3, "lt24": 3, "herd": [3, 7], "2407": [3, 5, 7, 8], "21783": [3, 7], "lwx": 3, "lin": [3, 5, 6, 7, 8, 9], "rui": [3, 5, 7, 9], "ruixuan": 3, "junbo": 3, "zhao": [3, 5, 7, 8], "ding": 3, "gang": [3, 5], "haobo": 3, "driven": [3, 5, 7, 8], "survei": [3, 5, 8, 9], "2406": [3, 5, 6, 7, 8], "15126": 3, "met24": 3, "owj": 3, "jeff": [3, 5, 8], "diogo": [3, 8], "almeida": [3, 8], "carrol": [3, 8], "wainwright": [3, 8], "pamela": [3, 5, 8], "mishkin": [3, 5, 8], "chong": [3, 8], "sandhini": [3, 8], "agarw": [3, 5, 8], "katarina": [3, 8], "slama": [3, 8], "alex": [3, 5, 7, 8], "rai": [3, 5, 7, 8], "john": [3, 5, 6, 8], "hilton": [3, 5, 7, 8], "fraser": [3, 8], "kelton": 3, "miller": [3, 5], "maddi": [3, 8], "simen": [3, 8], "peter": [3, 5, 7, 8], "welind": [3, 5, 8], "paul": [3, 5, 8], "christiano": [3, 8], "leik": [3, 5, 8], "ryan": [3, 5, 8], "2203": 3, "02155": 3, "qwe24": 3, "rsm": 3, "archit": 3, "sharma": [3, 8], "eric": [3, 5, 7, 8], "mitchel": [3, 6, 7], "stefano": [3, 5], "ermon": [3, 5], "man": [3, 5, 8], "chelsea": [3, 8], "finn": 3, "secretli": 3, "18290": 3, "swd": 3, "17": [3, 5, 6, 7, 8], "filip": [3, 8], "wolski": 3, "prafulla": 3, "dhariw": 3, "alec": [3, 5, 8], "radford": [3, 5, 8], "oleg": [3, 8], "klimov": 3, "1707": 3, "06347": 3, "smollm224": 3, "distil": [3, 4], "smollm2360mi24": 3, "sou24": 3, "html": [3, 6, 9], "srverh24": 3, "m\u00e1rton": 3, "daniel": [3, 5, 8], "rueckert": 3, "r\u00fcdiger": 3, "von": [3, 5, 7], "eisenhart": 3, "roth": [3, 5], "florian": 3, "hinterwimm": 3, "2411": [3, 6], "09539": 3, "tm": [3, 7], "23": [3, 5, 6, 7, 8], "hugo": [3, 7], "loui": [3, 5, 7], "martin": [3, 5, 6, 7, 8], "kevin": [3, 5, 7, 8], "stone": [3, 7], "albert": [3, 7], "amjad": [3, 7], "almahairi": [3, 7], "yasmin": [3, 7], "babaei": [3, 7], "nikolai": [3, 7], "bashlykov": [3, 7], "soumya": [3, 7], "batra": [3, 7], "prajjwal": [3, 7], "bhargava": [3, 7], "shruti": [3, 7], "bhosal": [3, 7], "dan": [3, 5, 7, 8, 9], "bikel": [3, 7], "luka": [3, 7], "blecher": [3, 7], "cristian": [3, 7], "canton": [3, 7], "ferrer": [3, 7], "moya": [3, 7], "guillem": [3, 7], "cucurul": [3, 7], "esiobu": [3, 7], "jude": [3, 7], "fernand": [3, 7], "jeremi": [3, 5, 6, 7], "fu": [3, 7], "wenyin": [3, 7], "brian": [3, 7, 8], "fuller": [3, 7, 8], "cynthia": [3, 7], "gao": [3, 5, 7, 8], "vedanuj": [3, 7], "goswami": [3, 7, 8], "naman": [3, 6, 7], "goyal": [3, 6, 7], "anthoni": [3, 6, 7], "hartshorn": [3, 7], "saghar": [3, 7], "hosseini": [3, 7], "hakan": [3, 7], "inan": [3, 7], "marcin": [3, 7], "karda": [3, 7], "viktor": [3, 7], "kerkez": [3, 7], "madian": [3, 7], "khabsa": [3, 7], "isabel": [3, 7, 8], "kloumann": [3, 7], "artem": [3, 7], "korenev": [3, 7], "punit": [3, 7], "singh": [3, 5, 6, 7], "koura": [3, 7], "mari": [3, 5, 7, 8], "ann": [3, 7, 8], "lachaux": [3, 7], "thibaut": [3, 7], "lavril": [3, 7], "jenya": [3, 7], "diana": [3, 5, 7], "liskovich": [3, 7], "yinghai": [3, 7], "yune": [3, 7], "mao": [3, 4, 7], "xavier": [3, 7], "martinet": [3, 7], "todor": [3, 7, 8], "mihaylov": [3, 7], "pushkar": [3, 7], "mishra": [3, 5, 7], "igor": [3, 5, 7, 8], "molybog": [3, 7], "yixin": [3, 5, 7], "nie": [3, 5, 6, 7], "andrew": [3, 5, 7, 8], "poulton": [3, 7], "reizenstein": [3, 7], "rashi": [3, 7], "rungta": [3, 7], "kalyan": [3, 7], "saladi": [3, 7], "alan": [3, 7, 8], "schelten": [3, 7], "ruan": [3, 7], "silva": [3, 7], "ranjan": [3, 7], "subramanian": [3, 7], "xiaoq": [3, 7], "ellen": [3, 7], "tan": [3, 5, 6, 7], "binh": [3, 7], "ross": [3, 4, 7, 8], "taylor": [3, 7], "adina": [3, 7, 8], "jian": [3, 5, 6, 7], "kuan": [3, 7], "puxin": [3, 7], "yan": [3, 4, 5, 7], "iliyan": [3, 7], "zarov": [3, 7], "yuchen": [3, 5, 7, 8], "angela": [3, 5, 7, 8], "fan": [3, 5, 7], "melani": [3, 7], "kambadur": [3, 7], "sharan": [3, 7], "narang": [3, 7], "aurelien": [3, 7], "rodriguez": [3, 7], "stojnic": [3, 7], "sergei": [3, 7], "edunov": [3, 7], "thoma": [3, 5, 7, 8], "scialom": [3, 7], "2307": [3, 7, 9], "09288": [3, 7], "vaa": [3, 8], "berti": [3, 8], "adarsh": [3, 8], "agraw": [3, 8], "ahm": [3, 8], "victor": [3, 8], "akinwand": [3, 8], "namir": [3, 8], "nuaimi": [3, 8], "najla": [3, 8], "alfaraj": [3, 8], "alhajjar": [3, 8], "aroyo": [3, 8], "trupti": [3, 8], "bavalatti": [3, 8], "max": [3, 5, 6, 8], "bartolo": [3, 8], "borhan": [3, 8], "blili": [3, 8], "hamelin": [3, 8], "kurt": [3, 8], "bollack": [3, 8], "rishi": [3, 5, 7, 8], "bomassani": [3, 8], "marisa": [3, 8], "ferrara": [3, 8], "boston": [3, 8], "sim\u00e9on": [3, 8], "campo": [3, 8], "kal": [3, 8], "chakra": [3, 8], "canyu": [3, 8], "codi": [3, 8], "coleman": [3, 8], "zachari": [3, 5, 8], "delpierr": [3, 8], "coudert": [3, 8], "leon": [3, 8], "derczynski": [3, 8], "debojyoti": [3, 8], "dutta": [3, 8], "ian": [3, 5, 8], "eisenberg": [3, 8], "ezick": [3, 8], "heather": [3, 8], "frase": [3, 8], "ram": [3, 7, 8], "gandikota": [3, 8], "agasthya": [3, 8], "gangavarapu": [3, 8], "ananya": [3, 5, 8], "geali": [3, 8], "rajat": [3, 8], "ghosh": [3, 5, 8], "goel": [3, 5, 8], "usman": [3, 8], "gohar": [3, 8], "sujata": [3, 8], "hale": [3, 8], "wiebk": [3, 8], "hutiri": [3, 8], "marvin": [3, 8], "imperi": [3, 8], "surgan": [3, 8], "jandial": [3, 8], "nick": [3, 5, 8], "judd": [3, 8], "felix": [3, 5, 8], "juefei": [3, 8], "fouts": [3, 8], "khomh": [3, 8], "bhavya": [3, 8], "kailkhura": [3, 8], "hannah": [3, 5, 8], "rose": [3, 8], "kirk": [3, 8], "klyman": [3, 8], "knotz": [3, 8], "kuchnik": [3, 8], "shachi": [3, 8], "kumar": [3, 5, 8], "srijan": [3, 8], "lengerich": [3, 8], "bo": [3, 5, 7, 8], "zeyi": [3, 8], "liao": [3, 5, 8], "eileen": [3, 8], "sarah": [3, 5, 8], "luger": [3, 8], "yifan": [3, 5, 8], "priyanka": [3, 8], "mammen": [3, 8], "kelvin": [3, 6, 8], "manyeki": [3, 8], "mcgregor": [3, 8], "virendra": [3, 8], "mehta": [3, 5, 8], "shafe": [3, 8], "moham": [3, 8], "moss": [3, 8], "lama": [3, 8], "nachman": [3, 8], "dinesh": [3, 8], "jinenh": [3, 8], "naganna": [3, 8], "amin": [3, 8], "nikanjam": [3, 8], "besmira": [3, 8], "nushi": [3, 8], "lui": [3, 5, 8], "oala": [3, 8], "iftach": [3, 8], "orr": [3, 5, 8], "alicia": [3, 5, 8], "parrish": [3, 5, 8], "cigdem": [3, 8], "patlak": [3, 8], "pietri": [3, 8], "forough": [3, 8], "poursabzi": [3, 8], "sangdeh": [3, 8], "eleonora": [3, 8], "presani": [3, 8], "fabrizio": [3, 8], "puletti": [3, 8], "r\u00f6ttger": [3, 8], "sahai": [3, 8], "santo": [3, 8], "nino": [3, 8], "scherrer": [3, 8], "alic": [3, 5, 8, 9], "schoenauer": [3, 8], "sebag": [3, 8], "patrick": [3, 6, 8], "schramowski": [3, 8], "abolfazl": [3, 8], "shahbazi": [3, 8], "vin": [3, 8], "xudong": [3, 5, 6, 8], "vamsi": [3, 8], "sistla": [3, 8], "leonard": [3, 8], "testuggin": [3, 8], "vithursan": [3, 8], "thangarasa": [3, 8], "elizabeth": [3, 5, 8], "watkin": [3, 8], "rebecca": [3, 5, 8], "weiss": [3, 8], "welti": [3, 8], "tyler": [3, 5, 8], "wilber": [3, 8], "jean": [3, 8], "poonam": [3, 8], "yadav": [3, 8], "xianjun": [3, 8], "yang": [3, 5, 6, 7, 8, 9], "yi": [3, 5, 6, 8, 9], "wenhui": [3, 8], "fedor": [3, 8], "zhdanov": [3, 8], "jiacheng": [3, 5, 8], "perci": [3, 5, 8], "liang": [3, 5, 8, 9], "mattson": [3, 8], "joaquin": [3, 8], "vanschoren": [3, 8], "v0": [3, 8], "12241": [3, 8], "wyg": 3, "tianhao": [3, 5, 7, 8], "weizh": 3, "yuan": [3, 5, 8], "olga": 3, "golovneva": 3, "jing": [3, 8], "yuandong": 3, "tian": 3, "jiantao": 3, "jiao": 3, "jason": [3, 5, 8], "weston": 3, "sainbayar": 3, "sukhbaatar": 3, "19594": 3, "xfg": 3, "shusheng": 3, "jiaxuan": 3, "wenji": 3, "ye": [3, 5, 7, 8, 9], "weilin": 3, "zhiyu": [3, 9], "mei": [3, 5, 7], "guangju": 3, "chao": 3, "10719": 3, "ywx": 3, "yueqin": 3, "zhendong": 3, "yujia": [3, 6], "xie": [3, 5], "mingyuan": 3, "paradigm": [3, 5], "semanticscholar": 3, "corpusid": 3, "270199610": 3, "suppos": [4, 5, 9], "econom": [4, 5, 6], "fuel": 4, "equival": [4, 5, 7], "consumpt": [4, 5], "contrari": 4, "truth": [4, 5, 7, 8, 9], "stanlei": 4, "jevon": 4, "a16z": 4, "andreessen": 4, "horowitz": 4, "10x": 4, "outpac": 4, "moor": 4, "pc": 4, "edholm": 4, "bandwidth": 4, "era": 4, "llmflation": 4, "mmlu": [4, 7, 8], "60": [4, 5, 6, 7, 8], "06": [4, 5, 6, 9], "price": [4, 5, 6, 7], "fallen": 4, "62": [4, 5, 7], "introduct": 4, "march": [4, 5, 9], "stem": [4, 5, 9], "compound": 4, "bit": [4, 6, 7], "tune": [4, 5, 8], "dpo": [4, 7], "competit": [4, 5, 6, 7, 8], "plummet": 4, "rapidli": [4, 7, 8, 9], "preciou": 4, "wouldn": [4, 5], "sens": [4, 8], "wait": [4, 5, 8], "wave": 4, "economist": 4, "1865": 4, "studi": [4, 9], "coal": 4, "industri": [4, 5, 6, 7, 8, 9], "made": [4, 5, 6, 7, 9], "counterintuit": 4, "discoveri": 4, "steam": 4, "spend": [4, 5, 6], "repeat": [4, 6], "didn": [4, 9], "smartphon": [4, 5, 6, 7], "server": [4, 5, 7, 9], "network": [4, 5, 6, 7, 9], "transmiss": 4, "got": 4, "cheaper": [4, 5], "shift": [4, 5, 6], "hd": 4, "stream": [4, 5, 6, 7, 9], "storag": [4, 5, 7, 8], "gigabyt": 4, "entir": [4, 5, 6, 7, 9], "massiv": [4, 5, 8], "broadli": [4, 6, 7, 9], "audio": [4, 5, 6], "transcript": [4, 6], "multimod": [4, 7, 8], "imag": [4, 5, 6, 7, 8], "exponenti": [4, 5], "growth": [4, 5, 6], "magnifi": 4, "everyth": [4, 9], "billion": [4, 5, 6, 7, 9], "dollar": [4, 5, 7], "annual": [4, 5, 6, 8], "millisecond": [4, 5], "latenc": [4, 5, 7, 8], "30": [4, 5, 6, 7, 8], "mobil": [4, 5, 7, 9], "b": [4, 5, 7, 8, 9], "tradeoff": [4, 7, 8, 9], "pro": [4, 5, 6, 7, 8], "trigger": [4, 6, 8], "premium": [4, 5], "innov": [4, 5, 6, 7, 8], "capac": [4, 5, 6, 7], "link": [4, 5], "dual": 4, "character": [4, 5, 8], "ahead": [4, 7, 8], "decai": [4, 7], "discuss": [4, 5, 7, 8], "area": [4, 5, 8, 9], "flash": [4, 6, 7], "cach": [4, 5, 6, 7], "prompt": [4, 5, 6, 8], "compress": [4, 5, 7], "provis": [4, 5], "extent": [4, 5, 8], "problema": 4, "accomplish": [4, 8, 9], "accompani": [4, 5, 8], "transact": [4, 5, 8], "roi": 4, "alloc": [4, 5, 6, 7, 8], "budget": [4, 7], "viabil": [4, 7], "prioriti": [4, 5, 7], "overlook": [4, 6], "thorough": [4, 7, 8], "identif": [4, 5], "specifi": [4, 5, 6, 7, 8, 9], "longev": 4, "accommod": 4, "evalu": [4, 6, 7, 9], "multi": [4, 5, 6, 7, 8, 9], "baselin": [4, 5, 7, 8], "met": [4, 5, 8], "equal": [4, 5, 6, 8], "concurr": [4, 7], "peak": 4, "spike": 4, "versu": [4, 5, 7, 8], "volum": [4, 5, 7, 8], "season": [4, 5], "variat": [4, 5, 7, 8], "uptim": 4, "mainten": [4, 5, 7, 8], "disrupt": [4, 5, 6], "backup": 4, "failov": 4, "clearli": [4, 5, 8, 9], "redund": [4, 5], "recoveri": [4, 5], "unexpect": [4, 5, 8, 9], "event": [4, 5], "seamless": [4, 5, 8], "broader": [4, 5, 6, 7, 8], "vector": [4, 7, 8], "augment": [4, 5, 7], "rag": [4, 7], "retent": [4, 5, 6], "polici": [4, 5, 6, 7], "essenti": [4, 5, 6, 7, 8, 9], "opportun": [4, 5, 6], "post": [4, 5, 7, 8], "32": [4, 5, 6, 7], "fp32": 4, "fp16": [4, 7], "proport": [4, 5, 7], "byte": 4, "120": [4, 5, 8], "gb": 4, "whole": [4, 5], "done": [4, 5, 7, 8, 9], "smollm2": [4, 5, 7, 9], "135m": [4, 7], "load_gguf": 4, "bartowski": 4, "gguf": [4, 7], "gguf_file_q2_k": 4, "q2_k": [4, 7], "gguf_file_f16": 4, "f16": 4, "model_q2_k": 4, "gguf_fil": 4, "model_f16": 4, "mlp": 4, "layer": [4, 5, 7, 9], "proxi": [4, 5, 6, 8], "mlp_weights_q2_k": 4, "gate_proj": 4, "mlp_weights_f16": 4, "tensor": [4, 9], "0145": 4, "1826": 4, "1377": 4, "1719": 4, "1387": 4, "0298": 4, "1631": 4, "0781": 4, "2051": [4, 5], "2070": 4, "0334": 4, "2891": 4, "1768": 4, "0488": 4, "2393": 4, "0396": 4, "1348": 4, "1533": 4, "0771": 4, "0845": 4, "0232": 4, "0178": 4, "1040": 4, "1582": 4, "1167": 4, "0474": 4, "0359": 4, "2500": 4, "0432": 4, "0972": 4, "0933": 4, "2188": 4, "0776": 4, "0674": 4, "requires_grad": 4, "0028": 4, "1852": 4, "1396": 4, "1506": 4, "1635": 4, "0043": 4, "0680": 4, "2257": 4, "1890": 4, "0464": 4, "2960": 4, "1840": 4, "0451": 4, "2395": 4, "0413": 4, "1446": 4, "0621": 4, "0478": 4, "0038": 4, "0830": 4, "1473": 4, "0926": 4, "0547": 4, "0824": 4, "0429": 4, "2737": 4, "0355": 4, "0782": 4, "2043": [4, 5], "0740": 4, "arriv": [4, 5], "pearson": 4, "numpi": [4, 5], "np": [4, 5], "arrai": [4, 8], "detach": 4, "graph": [4, 5], "weights_f16": 4, "weights_q2_k": 4, "flat_f16": 4, "flatten": 4, "flat_q2_k": 4, "corrcoef": 4, "4f": [4, 9], "9970": 4, "exemplifi": [4, 6, 7, 8], "70b": [4, 5, 7], "unsloth": 4, "141": 4, "q8_0": [4, 7], "75": [4, 8], "47": [4, 5, 7, 8], "cumul": [4, 5, 6], "26": [4, 5, 7], "19": [4, 5, 6, 7, 8], "space": [4, 5, 7, 8], "counterpart": 4, "spectrum": [4, 5, 6], "variant": [4, 5, 7, 8], "laptop": [4, 5], "desktop": [4, 5, 7], "enterpris": [4, 5, 7, 8, 9], "ceil": 4, "notabl": [4, 5, 8, 9], "bitnet": 4, "cpp": [4, 9], "arm": 4, "x86": 4, "speedup": [4, 7], "37x": 4, "07x": 4, "17x": 4, "beyond": [4, 5, 8], "raw": [4, 5, 7, 8, 9], "speed": [4, 5, 7, 8], "energi": [4, 5, 6], "55": [4, 5, 6, 7], "70": [4, 5, 7], "71": [4, 5], "82": [4, 8], "impress": [4, 7, 9], "100b": 4, "b1": 4, "58": [4, 6, 7], "pace": [4, 5, 6, 8], "second": [4, 5, 6, 7, 8], "kernel": 4, "characterist": [4, 5, 7, 8, 9], "excit": [4, 7], "frontier": [4, 8], "compel": [4, 5, 7, 9], "acceler": [4, 5, 7, 8], "faster": [4, 7], "arithmet": [4, 5], "benefici": [4, 5, 7], "sustain": [4, 5, 6, 7, 8], "Be": [4, 5, 7, 8], "fine": [4, 5, 8], "pure": [4, 5, 7, 9], "unlock": [4, 9], "track": [4, 5, 6, 8], "chargeback": 4, "regularli": [4, 5], "wz": 4, "jinheng": 4, "hansong": 4, "ting": [4, 8], "shaoguang": 4, "shume": 4, "ma": [4, 5, 8], "hongyu": [4, 5], "xia": [4, 5, 7], "infra": 4, "fast": [4, 5, 7, 8, 9], "lossless": 4, "16144": 4, "andreessenhorowitz24": 4, "huggingface4w": [4, 7], "2024w": [4, 7], "unsloth24": 4, "jonathan": [4, 5, 8], "ceo": [4, 5], "groq": [4, 7], "streamlin": [4, 5, 6, 7, 9], "notat": 4, "width": [4, 7], "_k": 4, "_0": 4, "matter": [5, 6], "beauti": 5, "smart": [5, 8], "agre": 5, "wrong": 5, "feynman": 5, "advent": 5, "verif": [5, 6, 7, 9], "norm": 5, "realm": 5, "convent": [5, 8], "evolut": [5, 7], "conceiv": 5, "entrench": 5, "seem": 5, "daunt": [5, 6], "ignor": 5, "outdat": [5, 6, 8, 9], "inevit": 5, "setback": 5, "imper": 5, "embrac": 5, "proactiv": [5, 8], "mindset": 5, "front": [5, 7], "incorpor": [5, 6, 7, 8, 9], "produc": [5, 6, 7, 8, 9], "novel": [5, 7], "ident": [5, 6], "isn": [5, 8], "bug": 5, "random": [5, 8, 9], "testabl": 5, "guarante": [5, 6, 7, 8, 9], "exceedingli": 5, "primari": [5, 8], "nucleu": 5, "2020": 5, "summari": [5, 6, 7, 8, 9], "alter": 5, "rigid": 5, "wildli": 5, "incoher": 5, "inadequ": [5, 8], "temp": 5, "df_result": 5, "ntemperatur": 5, "40": [5, 6, 7], "temp_respons": 5, "iterrow": [5, 8], "10000": [5, 6, 9], "appl": [5, 6, 9], "txt": [5, 6, 7, 9], "sec_fil": [5, 9], "nsecur": 5, "AND": [5, 9], "exchang": [5, 6, 8, 9], "commiss": [5, 6, 8, 9], "nwashington": 5, "20549": 5, "nform": 5, "pursuant": 5, "TO": [5, 8], "13": [5, 6, 7, 8], "OR": 5, "OF": [5, 8], "THE": [5, 8], "1934": 5, "nfor": 5, "fiscal": [5, 6], "septemb": [5, 6], "28": [5, 6, 7, 8], "nor": 5, "period": [5, 6, 8], "ncommiss": 5, "001": [5, 7], "36743": 5, "ng66145g66i43": 5, "jpg": 5, "nappl": 5, "exact": [5, 7, 8], "registr": 5, "charter": 5, "ncalifornia": 5, "t94": 5, "2404110": 5, "jurisdict": 5, "nof": 5, "employ": 5, "park": 5, "ncupertino": 5, "california": [5, 8, 9], "n95014": 5, "princip": 5, "offic": [5, 6, 8], "408": 5, "996": 5, "1010": 5, "telephon": 5, "regist": 5, "ntitl": 5, "ttrade": 5, "symbol": 5, "tname": 5, "ncommon": 5, "stock": [5, 9], "00001": 5, "naapl": 5, "tthe": 5, "nasdaq": [5, 6, 9], "llc": [5, 9], "n0": 5, "000": [5, 7, 9], "note": [5, 6, 7, 9], "2025": [5, 6], "875": 5, "625": 5, "2026": 5, "2027": 5, "375": 5, "2029": 5, "050": 5, "2031": [5, 8], "600": 5, "2042": 5, "nindic": 5, "issuer": 5, "405": 5, "nye": 5, "preced": [5, 9], "shorter": 5, "past": [5, 6, 8], "90": [5, 6, 7, 8], "submit": [5, 7, 8], "electron": 5, "232": 5, "filer": 5, "12b": [5, 8], "nlarg": 5, "tacceler": 5, "nnon": 5, "tsmaller": 5, "nemerg": 5, "nif": 5, "elect": [5, 8], "revis": [5, 8], "attest": 5, "404": 5, "sarban": 5, "oxlei": 5, "7262": 5, "firm": [5, 8], "prepar": [5, 6, 7, 8], "correct": [5, 6, 8], "restat": 5, "incent": 5, "compens": 5, "240": 5, "10d": 5, "shell": 5, "aggreg": [5, 8, 9], "vote": 5, "held": [5, 9], "affili": [5, 9], "29": [5, 7, 8, 9], "last": [5, 6, 8, 9], "quarter": 5, "628": [5, 9], "553": [5, 9], "sole": [5, 8], "disclosur": [5, 6, 7, 8], "director": [5, 7, 8], "date": [5, 6], "exclud": 5, "n15": 5, "115": [5, 9], "823": [5, 9], "outstand": [5, 9], "octob": [5, 9], "18": [5, 6, 7, 8, 9], "ndocument": 5, "BY": 5, "nportion": 5, "meet": [5, 6, 8, 9], "sharehold": [5, 6], "iii": 5, "ntabl": 5, "npage": 5, "npart": 5, "nitem": 5, "nbusi": 5, "1a": 5, "nrisk": 5, "1b": [5, 7, 8], "nunresolv": 5, "staff": 5, "comment": 5, "n17": 5, "1c": 5, "ncybersecur": 5, "nproperti": 5, "n18": 5, "nlegal": 5, "proceed": [5, 6, 8], "nmine": 5, "ii": [5, 7, 9], "nmarket": 5, "stockhold": 5, "purchas": [5, 6, 8], "n19": 5, "reserv": [5, 6], "n20": 5, "nmanag": 5, "n21": 5, "7a": 5, "nquantit": 5, "n27": 5, "nfinanci": 5, "supplementari": 5, "n28": 5, "nchang": 5, "disagr": 5, "n51": 5, "9a": 5, "ncontrol": 5, "procedur": [5, 6, 8], "9b": 5, "nother": 5, "n52": 5, "9c": 5, "ndisclosur": 5, "foreign": [5, 6], "ndirector": 5, "corpor": [5, 6, 8], "nexecut": 5, "ownership": [5, 7], "certain": [5, 6, 8, 9], "owner": 5, "ncertain": 5, "nprincip": 5, "fee": [5, 6], "iv": 5, "nexhibit": 5, "n53": 5, "n56": 5, "nthi": 5, "litig": [5, 6, 7], "reform": 5, "1995": 5, "uncertainti": [5, 6, 7, 8], "macroeconom": [5, 6], "anticip": [5, 6, 8], "intend": [5, 7, 8], "caus": [5, 8, 9], "oblig": [5, 6], "nunless": 5, "herein": 5, "calendar": 5, "wholli": 5, "subsidiari": 5, "unless": [5, 7], "ncompani": 5, "manufactur": 5, "tablet": [5, 6, 7], "wearabl": 5, "accessori": 5, "sell": [5, 8], "varieti": [5, 7], "52": [5, 8], "53": [5, 6, 8], "week": 5, "saturdai": 5, "nproduct": 5, "niphon": 5, "io": [5, 6, 9], "iphon": [5, 6], "se": [5, 8], "nmac": 5, "maco": [5, 7], "mac": [5, 7], "macbook": 5, "air": 5, "imac": 5, "studio": 5, "nipad": 5, "multipurpos": 5, "ipado": 5, "ipad": 5, "nwearabl": 5, "home": [5, 6, 9], "smartwatch": 5, "wireless": 5, "headphon": 5, "spatial": 5, "watcho": 5, "watch": 5, "ultra": 5, "airpod": 5, "beat": [5, 7], "visiono": 5, "nhome": 5, "tv": 5, "tvo": 5, "homepod": 5, "fidel": [5, 9], "naccessori": 5, "brand": 5, "third": [5, 6, 7, 8], "parti": [5, 6, 7, 8], "nservic": 5, "nadvertis": 5, "advertis": 5, "licens": [5, 6], "napplecar": 5, "portfolio": [5, 6], "applecar": 5, "repair": 5, "coverag": [5, 6, 8], "accident": 5, "damag": [5, 8], "theft": [5, 8], "ncloud": 5, "ndigit": 5, "app": [5, 6, 7], "discov": [5, 7, 8], "download": [5, 7], "music": 5, "podcast": 5, "subscript": [5, 7], "arcad": 5, "sm": 5, "listen": [5, 7], "radio": 5, "station": 5, "magazin": 5, "exclus": 5, "sport": 5, "npayment": 5, "payment": 5, "credit": [5, 6], "pai": [5, 7], "cashless": 5, "nsegment": 5, "primarili": [5, 6, 8], "geograph": [5, 6, 8], "basi": [5, 7], "segment": [5, 6, 8, 9], "america": [5, 6], "europ": 5, "china": [5, 6, 7, 8], "japan": 5, "rest": [5, 7], "asia": 5, "pacif": 5, "north": [5, 8], "south": 5, "european": [5, 8], "india": 5, "middl": [5, 7, 8], "east": 5, "africa": 5, "mainland": 5, "kong": 5, "taiwan": 5, "australia": 5, "asian": [5, 6], "although": [5, 7], "partner": [5, 6, 7, 8], "mid": [5, 6], "resel": [5, 6], "retail": 5, "sale": [5, 6], "indirect": 5, "channel": [5, 8], "cellular": 5, "carrier": 5, "net": [5, 6, 9], "38": [5, 6, 7, 8], "ncompetit": 5, "downward": 5, "pressur": [5, 8], "gross": [5, 8], "cycl": [5, 8], "competitor": [5, 6, 7, 8], "compet": [5, 6, 7], "imit": 5, "infring": [5, 7], "intellectu": [5, 7, 8], "marketplac": [5, 8], "nearli": [5, 7], "reput": [5, 8], "expand": [5, 6, 7, 8], "profit": [5, 6, 8, 9], "illegitim": [5, 8], "collabor": [5, 7, 8], "nsuppli": 5, "nalthough": 5, "particip": 5, "shortag": 5, "commod": [5, 6, 7], "fluctuat": [5, 6], "commonli": 5, "until": [5, 8, 9], "supplier": 5, "matur": 5, "concentr": [5, 6], "enter": [5, 9], "agreement": [5, 6], "suppli": [5, 6, 9], "renew": [5, 6], "nresearch": 5, "nbecaus": 5, "upon": [5, 6, 8], "flow": [5, 6, 9], "acquisit": [5, 6, 8], "nintellectu": 5, "broad": [5, 6, 7, 9], "patent": 5, "copyright": [5, 7], "trademark": 5, "secret": 5, "differenti": 5, "skill": [5, 8], "personnel": 5, "pursu": [5, 8], "thousand": [5, 7], "durat": 5, "adequ": [5, 8], "nin": 5, "holidai": [5, 8], "fill": 5, "inventori": 5, "older": [5, 7], "newer": 5, "distributor": 5, "nhuman": 5, "strive": 5, "retain": [5, 6, 7, 8], "talent": [5, 6], "member": [5, 8], "164": 5, "ncompens": 5, "equit": 5, "succe": 5, "health": [5, 6, 8], "awai": [5, 6, 8], "ngrowth": 5, "career": 5, "leadership": [5, 8], "nworkplac": 5, "workplac": 5, "ninclus": 5, "workforc": 5, "nengag": 5, "among": [5, 7, 8, 9], "everyon": [5, 7], "gaug": 5, "sentiment": [5, 6, 7, 9], "nhealth": 5, "everywher": 5, "crisi": 5, "visitor": 5, "navail": 5, "quarterli": 5, "q": [5, 6, 7, 8], "amend": 5, "sec": [5, 6, 9], "Such": [5, 8], "charg": 5, "investor": [5, 6, 9], "aspx": 5, "websit": [5, 7, 8], "environment": [5, 8], "referenc": [5, 6], "inact": 5, "textual": 5, "unknown": [5, 6, 8], "advers": 5, "conjunct": 5, "consolid": [5, 6], "nmacroeconom": 5, "facil": 5, "assembli": 5, "site": [5, 9], "nadvers": 5, "slow": 5, "recess": 5, "unemploy": [5, 6], "inflat": [5, 6], "tighter": 5, "currenc": [5, 6], "monetari": 5, "contract": [5, 7], "logist": 5, "instabl": [5, 8], "inabl": [5, 6], "financ": [5, 7, 8], "insolv": 5, "counterparti": 5, "debt": 5, "liquid": [5, 6], "fair": [5, 8], "instrument": 5, "polit": [5, 8], "disput": 5, "geopolit": 5, "tension": [5, 8], "terror": 5, "accid": 5, "interrupt": 5, "npolit": 5, "outsourc": [5, 6], "korea": 5, "vietnam": 5, "restrict": [5, 7, 8, 9], "tariff": 5, "export": [5, 6], "portion": [5, 7], "revenu": [5, 6, 9], "restructur": 5, "ceas": 5, "escal": [5, 8], "nmani": 5, "prone": [5, 8], "earthquak": 5, "climat": 5, "weather": 5, "plant": 5, "terrorist": [5, 8], "attack": [5, 8], "hostil": 5, "ransomwar": 5, "cybersecur": [5, 6, 8], "labor": 5, "nsuch": 5, "imposs": [5, 7], "slowdown": 5, "outag": 5, "neg": [5, 6, 8, 9], "pandem": 5, "covid": 5, "economi": 5, "imposit": 5, "stringent": [5, 7, 8], "travel": 5, "freight": 5, "movement": 5, "ramp": 5, "nfollow": 5, "expenditur": 5, "resum": 5, "exacerb": 5, "insur": 5, "nglobal": 5, "unabl": 5, "assur": [5, 8], "minor": [5, 6, 8], "naddition": 5, "intensifi": 5, "seamlessli": 5, "nto": 5, "stimul": 5, "ndue": 5, "upgrad": 5, "quantiti": 5, "defect": 5, "defici": 5, "supersed": 5, "nsubstanti": 5, "transport": 5, "reimburs": 5, "warranti": 5, "unanticip": 5, "liabil": 5, "final": [5, 6, 8, 9], "finish": [5, 8], "destin": 5, "prepay": 5, "termin": [5, 7], "recover": 5, "exposur": [5, 8], "nfutur": 5, "semiconductor": 5, "suffer": [5, 8], "constrain": [5, 7, 9], "shipment": 5, "unexpectedli": 5, "interfer": 5, "unsaf": [5, 8], "expos": [5, 6, 8], "widespread": [5, 8], "vulner": [5, 6, 8], "compromis": [5, 7, 8], "claim": [5, 6, 7, 8], "intang": 5, "lost": [5, 6, 8], "cancel": 5, "obsolet": 5, "exce": [5, 8], "realiz": 5, "accru": 5, "excess": 5, "impair": 5, "whenev": 5, "circumst": 5, "amount": [5, 6, 8, 9], "carri": [5, 7, 9], "incur": 5, "unpredict": [5, 8], "obsolesc": 5, "forecast": [5, 6, 8], "incorrectli": [5, 8, 9], "extens": [5, 7, 9], "issuanc": 5, "unknowingli": [5, 8], "notifi": 5, "preclud": 5, "bui": 5, "percept": 5, "android": [5, 6], "playstat": 5, "nintendo": 5, "xbox": 5, "inclin": 5, "devot": 5, "dissatisfi": 5, "vast": [5, 8], "storefront": 5, "safari": 5, "union": [5, 8], "eu": [5, 6, 8], "dma": [5, 6], "narrow": [5, 7, 8], "scope": [5, 6, 7, 8], "elimin": [5, 7], "nfailur": 5, "appeal": [5, 6], "subscrib": [5, 6], "nsome": 5, "manner": [5, 6, 8], "nurtur": 5, "nmuch": 5, "chief": [5, 6], "silicon": 5, "vallei": 5, "constantli": 5, "driver": [5, 7], "recruit": 5, "subsidi": 5, "staf": 5, "contractor": 5, "placement": 5, "increment": 5, "weaken": 5, "telecommun": 5, "war": 5, "virus": 5, "ins": 5, "incid": [5, 8], "ineffect": 5, "thing": [5, 9], "interf": 5, "imped": 5, "ship": 5, "nloss": 5, "unauthor": [5, 8], "confidenti": [5, 7], "encrypt": 5, "But": [5, 8, 9], "behalf": 5, "normal": [5, 6, 8, 9], "investig": [5, 8], "penalti": [5, 7], "frequenc": [5, 7, 8], "actor": [5, 8], "circumv": [5, 8], "obfusc": 5, "forens": 5, "hinder": [5, 9], "recov": 5, "perpetr": 5, "profil": [5, 7], "authent": 5, "hack": [5, 8], "malfeas": 5, "faulti": 5, "password": 5, "irregular": 5, "fraudul": 5, "induc": 5, "disclos": [5, 6, 9], "usernam": 5, "turn": [5, 6, 8, 9], "multifactor": 5, "unusu": 5, "freez": 5, "suspici": 5, "nwhile": 5, "ninvest": 5, "ongo": [5, 6, 7], "contempl": 5, "endeavor": 5, "distract": 5, "tangibl": 5, "approv": 5, "oner": 5, "ventur": 5, "riski": 5, "leas": 5, "unfavor": [5, 6], "arisen": 5, "ordinari": 5, "cours": [5, 7, 8], "resolv": [5, 7, 8], "sometim": 5, "indemnif": 5, "indemnifi": 5, "alleg": 5, "magnitud": 5, "assert": [5, 6], "royalti": 5, "vigor": 5, "defend": 5, "court": [5, 7], "internation": 5, "plaintiff": 5, "injunct": 5, "relief": 5, "nregardless": 5, "merit": 5, "recognit": [5, 7, 8], "settl": 5, "uncertain": [5, 6], "disgorg": 5, "remedi": [5, 8], "worldwid": 5, "antitrust": [5, 6], "bill": [5, 6], "commerc": 5, "televis": 5, "film": 5, "anticorrupt": 5, "cash": [5, 6], "repatri": 5, "launder": 5, "tax": [5, 6], "wast": 5, "recycl": 5, "ncomplianc": 5, "impos": [5, 7, 8, 9], "agent": [5, 7, 8], "nregulatori": 5, "ban": [5, 8], "nexpect": 5, "increasingli": [5, 7, 8, 9], "greenhous": 5, "ga": 5, "emiss": 5, "civil": 5, "disagre": 5, "perceiv": 5, "feder": 5, "nfrom": 5, "noncompli": 5, "individu": [5, 6, 7, 8], "lawsuit": [5, 7], "monopol": 5, "nfurther": 5, "earn": 5, "search": [5, 6, 7, 8], "nthere": 5, "transfer": 5, "pass": [5, 6, 7, 8, 9], "pend": 5, "inquiri": [5, 8], "government": 5, "entiti": [5, 7, 8, 9], "biometr": 5, "notif": 5, "permit": [5, 7, 9], "healthcar": [5, 6, 7], "liabl": 5, "investigatori": 5, "cardhold": 5, "acquir": 5, "denomin": 5, "offset": 5, "strengthen": [5, 8], "nconvers": 5, "thu": 5, "hedg": 5, "deterior": 5, "sovereign": 5, "heighten": [5, 8], "worsen": 5, "A": [5, 6, 7, 8, 9], "collater": 5, "bank": 5, "unsecur": 5, "subassembli": 5, "assembl": 5, "legisl": 5, "ireland": [5, 8], "singapor": 5, "organis": 5, "statutori": 5, "valuat": [5, 6], "defer": 5, "bodi": [5, 8], "adequaci": 5, "ow": 5, "ngener": 5, "repurchas": 5, "dividend": 5, "consumm": 5, "declar": [5, 6], "board": [5, 6, 8], "unresolv": 5, "nnone": 5, "threat": [5, 6, 8], "postur": 5, "25": [5, 6, 7, 8], "2016": 5, "coordin": [5, 8], "committe": [5, 8], "oversight": [5, 8], "counsel": 5, "chair": 5, "headquart": 5, "cupertino": [5, 9], "center": [5, 8, 9], "formal": [5, 8, 9], "conclud": [5, 7], "uninstal": 5, "web": [5, 6, 7, 8], "browser": 5, "june": 5, "contractu": 5, "desist": 5, "stai": [5, 7], "grant": 5, "ndepart": 5, "justic": 5, "depart": [5, 8], "doj": 5, "district": 5, "attornei": 5, "jersei": 5, "redress": [5, 8], "anticompetit": 5, "nonmonetari": 5, "defens": [5, 8], "nepic": 5, "epic": 5, "northern": 5, "unfair": [5, 8], "enjoin": 5, "extern": [5, 6, 8], "januari": 5, "motion": 5, "oppos": [5, 8], "vacat": 5, "fourth": 5, "mine": 5, "nnot": 5, "aapl": 5, "nholder": 5, "na": [5, 8], "301": 5, "npurchas": 5, "nshare": 5, "nperiod": 5, "ttotal": 5, "taverag": 5, "npaid": 5, "nannounc": 5, "napproxim": 5, "That": [5, 6, 8, 9], "nunder": 5, "njune": 5, "august": [5, 6, 8], "nopen": 5, "negoti": [5, 8], "t35": 5, "697": 5, "t224": 5, "naugust": 5, "31": [5, 6, 7], "t42": 5, "910": 5, "t221": 5, "39": [5, 6, 7], "nseptemb": 5, "t33": 5, "653": 5, "t222": 5, "86": [5, 7], "ntotal": [5, 8], "t112": 5, "260": 5, "t89": 5, "074": 5, "110": 5, "10b5": 5, "reinvest": 5, "dow": 5, "supersector": 5, "27": [5, 7, 8], "2019": 5, "n2218": 5, "tseptemb": 5, "t100": 5, "t207": 5, "t273": 5, "t281": 5, "t322": 5, "t430": 5, "t113": 5, "t156": 5, "t131": 5, "t155": 5, "t210": 5, "ndow": 5, "t146": 5, "t216": 5, "t215": 5, "nfirst": 5, "nsecond": 5, "nthird": 5, "sequoia": 5, "nfourth": 5, "plu": [5, 7], "nfiscal": 5, "six": 5, "realign": 5, "span": [5, 7, 8], "indirectli": 5, "n2024": 5, "tchang": 5, "t2023": 5, "t2022": 5, "namerica": 5, "t167": 5, "045": 5, "t3": 5, "t162": 5, "560": 5, "t169": 5, "658": 5, "neurop": 5, "t101": 5, "328": 5, "t7": 5, "294": 5, "t95": 5, "118": 5, "ngreater": 5, "t66": 5, "952": 5, "t72": 5, "559": 5, "t74": 5, "njapan": 5, "t25": 5, "052": 5, "t24": 5, "257": 5, "977": 5, "nrest": 5, "t30": 5, "t4": 5, "t29": 5, "615": 5, "t1": 5, "t391": 5, "035": 5, "t2": 5, "t383": 5, "285": 5, "t394": 5, "weak": [5, 6, 8], "renminbi": 5, "yen": [5, 9], "t201": 5, "183": 5, "t200": 5, "583": 5, "t205": 5, "489": 5, "984": 5, "357": 5, "t40": 5, "177": [5, 8], "t26": 5, "694": 5, "t28": 5, "300": 5, "292": 5, "t37": 5, "005": 5, "t39": 5, "845": [5, 8], "t41": 5, "241": 5, "n96": 5, "169": 5, "t13": 5, "t85": 5, "t9": 5, "t78": 5, "129": [5, 8], "amort": 5, "bundl": 5, "flat": [5, 6], "ngross": 5, "t109": 5, "633": 5, "t108": 5, "803": 5, "t114": 5, "728": 5, "t71": 5, "t60": 5, "345": 5, "t56": 5, "054": 5, "t180": 5, "683": 5, "148": 5, "t170": 5, "782": 5, "t36": 5, "t73": 5, "t70": 5, "t46": 5, "t44": 5, "t43": 5, "noper": 5, "t31": 5, "370": 5, "t5": 5, "915": 5, "t14": 5, "251": 5, "npercentag": 5, "t8": 5, "nsell": 5, "administr": 5, "097": 5, "932": 5, "094": 5, "t6": 5, "t57": 5, "467": 5, "t54": 5, "847": 5, "t51": 5, "t15": 5, "headcount": 5, "nprovis": 5, "749": 5, "t16": 5, "741": 5, "t19": 5, "neffect": 5, "nstatutori": 5, "t21": 5, "aid": [5, 8], "nliquid": 5, "unrestrict": 5, "140": 5, "ndebt": 5, "97": [5, 6, 8], "payabl": 5, "promissori": 5, "nleas": 5, "nmanufactur": 5, "noncancel": 5, "ndeem": 5, "tcja": 5, "nstate": 5, "fund": [5, 6, 7], "escrow": 5, "ncapit": 5, "95": [5, 8], "nrecent": 5, "pronounc": 5, "nincom": 5, "fasb": 5, "asu": 5, "09": [5, 6, 8], "740": 5, "reconcili": [5, 6], "reconcil": [5, 9], "disaggreg": 5, "prospect": 5, "novemb": [5, 8], "07": [5, 6, 8, 9], "280": 5, "maker": 5, "codm": 5, "retrospect": 5, "ncritic": 5, "conform": [5, 9], "gaap": 5, "nuncertain": 5, "domest": 5, "taxat": 5, "resolut": [5, 6], "conting": 5, "ninterest": 5, "forth": 5, "hypothet": 5, "nsensit": 5, "nhypothet": 5, "nrate": 5, "npotenti": 5, "n100": 5, "tenor": 5, "ndeclin": 5, "755": 5, "089": 5, "nterm": 5, "nincreas": 5, "t139": 5, "t194": 5, "nforeign": 5, "var": 5, "mont": 5, "carlo": 5, "interv": 5, "538": 5, "669": 5, "nindex": 5, "tpage": 5, "nconsolid": 5, "n29": 5, "n30": 5, "sheet": 5, "n31": 5, "n32": 5, "n33": 5, "nnote": 5, "n34": 5, "nreport": 5, "n48": 5, "nall": 5, "omit": 5, "submiss": 5, "nyear": 5, "n2023": 5, "n2022": 5, "nnet": 5, "t294": 5, "866": 5, "t298": 5, "085": 5, "t316": 5, "199": 5, "t96": 5, "ncost": 5, "t185": 5, "233": 5, "t189": 5, "282": 5, "471": 5, "119": 5, "855": 5, "t22": 5, "075": 5, "352": 5, "t214": 5, "137": 5, "t223": 5, "546": 5, "t123": 5, "216": 5, "t119": 5, "437": 5, "t269": 5, "565": 5, "334": 5, "485": 5, "736": 5, "103": 5, "t93": 5, "995": 5, "t99": 5, "nearn": 5, "nbasic": 5, "ndilut": 5, "08": [5, 7, 9], "343": [5, 8], "783": 5, "744": 5, "215": 5, "963": 5, "095": 5, "812": 5, "547": 5, "325": 5, "819": 5, "nsee": 5, "translat": [5, 7, 8], "t395": 5, "765": 5, "511": 5, "unreal": 5, "832": 5, "t323": 5, "212": 5, "nadjust": 5, "337": 5, "717": 5, "394": 5, "138": 5, "850": 5, "563": 5, "104": 5, "t204": 5, "t253": 5, "816": 5, "899": 5, "272": 5, "t98": 5, "016": 5, "652": 5, "t88": 5, "531": 5, "nasset": 5, "ncurrent": 5, "ncash": 5, "943": 5, "965": 5, "228": 5, "590": 5, "naccount": 5, "410": 5, "508": 5, "nvendor": 5, "t32": 5, "833": 5, "477": 5, "ninventori": 5, "286": 5, "331": 5, "287": 5, "695": 5, "t152": 5, "987": 5, "t143": 5, "566": 5, "t91": 5, "479": 5, "544": 5, "t45": 5, "680": 5, "715": 5, "834": 5, "t64": 5, "758": 5, "t211": 5, "993": 5, "t209": 5, "017": 5, "t364": 5, "980": [5, 8], "t352": 5, "nliabil": 5, "t68": 5, "960": 5, "t62": 5, "611": 5, "304": 5, "t58": 5, "829": 5, "ndefer": 5, "249": 5, "061": 5, "ncommerci": 5, "967": 5, "985": 5, "t10": 5, "912": 5, "822": 5, "t176": 5, "392": 5, "t145": 5, "308": 5, "750": 5, "888": 5, "t49": 5, "848": 5, "638": 5, "t308": 5, "030": [5, 7], "t290": 5, "ncommit": 5, "nsharehold": 5, "400": [5, 6], "116": 5, "786": 5, "550": 5, "n83": 5, "276": 5, "naccumul": 5, "deficit": 5, "154": 5, "214": 5, "172": 5, "452": 5, "950": 5, "146": [5, 8], "t50": 5, "672": 5, "t63": 5, "090": 5, "nbegin": 5, "849": 5, "365": 5, "423": 5, "346": [5, 6], "175": 5, "withheld": 5, "settlement": 5, "521": 5, "971": 5, "t12": 5, "034": 5, "t11": 5, "nend": 5, "t83": 5, "nretain": 5, "068": 5, "562": 5, "ndividend": 5, "218": 5, "793": 5, "612": 5, "099": 5, "454": 5, "846": 5, "77": [5, 6, 7], "046": 5, "186": 5, "109": 5, "t163": 5, "rsu": 5, "t0": 5, "98": [5, 6, 7], "94": [5, 6, 7, 8], "737": 5, "929": 5, "ndepreci": 5, "445": 5, "519": 5, "688": 5, "038": 5, "266": 5, "227": 5, "006": 5, "788": 5, "356": 5, "271": 5, "520": 5, "618": 5, "484": 5, "731": 5, "684": 5, "499": 5, "020": 5, "889": 5, "448": 5, "552": 5, "031": 5, "t118": 5, "254": 5, "t110": 5, "543": 5, "t122": 5, "151": 5, "48": [5, 7], "656": 5, "513": 5, "76": [5, 8], "923": 5, "nproce": 5, "211": 5, "686": 5, "917": 5, "135": 5, "828": [5, 6], "446": 5, "447": 5, "959": 5, "708": 5, "086": 5, "935": 5, "705": 5, "354": 5, "nfinanc": 5, "441": 5, "431": 5, "223": [5, 8], "234": [5, 8], "025": 5, "841": 5, "nrepurchas": 5, "949": 5, "89": [5, 8], "402": 5, "465": 5, "nrepay": 5, "958": 5, "repay": 5, "978": 5, "955": 5, "361": 5, "581": 5, "160": 5, "121": 5, "983": 5, "488": 5, "794": 5, "760": 5, "nsupplement": 5, "102": 5, "t18": 5, "679": 5, "573": 5, "33": [5, 6, 7, 8], "nbasi": 5, "prior": [5, 8], "reclassifi": 5, "nrevenu": 5, "remit": [5, 8], "straight": 5, "vest": 5, "sold": 5, "nderiv": 5, "nonleas": 5, "34": [5, 6, 8], "entitl": 5, "commenc": 5, "deliveri": 5, "stand": 5, "ssp": 5, "icloud": 5, "siri": 5, "discount": 5, "undeliv": 5, "unbil": 5, "n26": 5, "n37": 5, "moder": [5, 7], "64": [5, 7, 8], "dilut": 5, "nnumer": 5, "ndenomin": 5, "nweight": 5, "312": 5, "316": 5, "856": 5, "antidilut": 5, "tunreal": 5, "ngain": 5, "tfair": 5, "nvalu": 5, "tcash": 5, "nequival": 5, "tcurrent": 5, "tnon": 5, "t27": 5, "nlevel": 5, "nmonei": 5, "t778": 5, "nmutual": 5, "n515": 5, "t105": 5, "t617": 5, "nsubtot": 5, "293": 5, "395": 5, "nu": 5, "treasuri": 5, "516": 5, "t212": 5, "087": 5, "380": 5, "159": 5, "t703": 5, "t17": 5, "568": 5, "158": 5, "810": 5, "ncertif": 5, "deposit": 5, "t873": 5, "t387": 5, "t478": 5, "066": 5, "ncorpor": 5, "t65": 5, "622": 5, "t270": 5, "953": 5, "939": 5, "027": 5, "t47": 5, "886": 5, "nmunicip": 5, "t412": 5, "t405": 5, "t190": 5, "nmortgag": 5, "595": 5, "t175": 5, "403": 5, "t23": 5, "367": 5, "278": [5, 8], "t132": 5, "t583": 5, "635": 5, "t128": 5, "056": 5, "966": 5, "t34": 5, "t160": 5, "t688": 5, "650": 5, "36": [5, 6, 7, 8], "359": [5, 8], "t481": 5, "n442": 5, "t428": 5, "t923": 5, "t909": 5, "406": 5, "114": 5, "468": 5, "136": 5, "t271": 5, "533": 5, "048": [5, 7], "491": 5, "332": 5, "t320": 5, "t608": 5, "t76": 5, "840": 5, "956": 5, "890": 5, "t20": 5, "627": 5, "243": 5, "t628": 5, "t602": 5, "t192": 5, "t410": 5, "735": 5, "636": 5, "t344": 5, "t144": 5, "470": 5, "657": 5, "831": 5, "125": 5, "162": 5, "t173": 5, "752": 5, "corrobor": 5, "mortgag": [5, 6], "classifi": [5, 8], "37": [5, 7, 8], "swap": 5, "remeasur": 5, "notion": 5, "069": 5, "730": 5, "575": 5, "493": 5, "t104": 5, "777": 5, "nhedg": 5, "433": 5, "505": 5, "247": [5, 8], "ntrade": 5, "41": [5, 7, 8], "44": [5, 8], "depreci": 5, "nland": 5, "690": 5, "nmachineri": 5, "t80": 5, "205": [5, 7], "314": 5, "nleasehold": 5, "839": 5, "599": 5, "73": [5, 7, 8], "884": 5, "852": 5, "t55": 5, "906": 5, "601": 5, "703": 5, "010": 5, "457": 5, "634": 5, "391": 5, "neuropean": 5, "opinion": [5, 6, 8], "1991": 5, "2007": 5, "irish": 5, "branch": 5, "2003": 5, "2014": 5, "2015": 5, "minist": 5, "juli": [5, 8], "annul": 5, "ecj": 5, "hear": 5, "asid": 5, "confirm": 5, "unrecogn": [5, 6], "nfeder": 5, "571": 5, "080": 5, "644": 5, "265": 5, "801": 5, "726": 5, "570": 5, "298": 5, "49": [5, 6, 8], "t84": 5, "428": 5, "603": 5, "483": [5, 8], "t347": 5, "t669": 5, "076": 5, "830": 5, "419": 5, "072": 5, "pretax": 5, "72": [5, 6, 8], "ncomput": 5, "885": 5, "012": 5, "124": 5, "518": 5, "nimpact": 5, "246": 5, "311": 5, "366": 5, "397": 5, "nexcess": 5, "893": 5, "871": 5, "192": [5, 8], "739": 5, "ntax": 5, "carryforward": 5, "302": 5, "naccru": 5, "413": [5, 8], "421": 5, "nunreal": 5, "173": 5, "168": 5, "873": 5, "743": 5, "nless": 5, "374": 5, "007": 5, "369": 5, "551": 5, "998": 5, "nright": 5, "179": 5, "nminimum": 5, "674": 5, "940": 5, "t511": 5, "t455": 5, "t490": 5, "805": 5, "202": 5, "indefinit": 5, "temporari": 5, "727": 5, "044": 5, "284": 5, "ndecreas": 5, "386": 5, "463": 5, "982": 5, "542": 5, "936": 5, "070": 5, "expir": 5, "statut": 5, "229": 5, "494": 5, "closur": 5, "intercompani": 5, "exceed": [5, 8], "multiyear": 5, "exercis": 5, "noncash": 5, "rou": 5, "tfinanci": 5, "t2024": 5, "tother": 5, "661": 5, "tproperti": 5, "015": 5, "303": 5, "676": 5, "t165": 5, "t752": 5, "t859": 5, "430": 5, "842": [5, 8], "tfinanc": 5, "n2025": 5, "820": 5, "t171": 5, "991": 5, "n2026": 5, "914": 5, "n2027": 5, "t59": 5, "733": 5, "n2028": 5, "360": 5, "t38": 5, "398": 5, "n2029": 5, "187": 5, "nthereaft": 5, "t837": 5, "undiscount": 5, "790": 5, "imput": 5, "376": 5, "534": 5, "t896": 5, "borrow": 5, "proce": 5, "nine": [5, 8], "nmatur": 5, "333": 5, "264": 5, "948": 5, "645": 5, "309": 5, "arrear": 5, "namount": 5, "n2013": 5, "nfix": 5, "2062": 5, "t97": 5, "341": 5, "03": [5, 6], "65": [5, 8], "t106": 5, "572": 5, "n97": 5, "nunamort": 5, "321": 5, "358": 5, "113": 5, "662": 5, "930": 5, "342": 5, "800": 5, "180": 5, "88": [5, 6], "ndure": 5, "425": 5, "426": 5, "372": 5, "589": 5, "055": 5, "appreci": 5, "four": [5, 7, 8], "holder": [5, 7], "n2014": 5, "bonu": 5, "nrestrict": 5, "nnumber": 5, "nrsu": 5, "ngrant": 5, "naggreg": 5, "nfair": 5, "nbalanc": 5, "t240": 5, "427": [5, 8], "t75": 5, "t150": 5, "861": 5, "501": 5, "768": 5, "87": [5, 6, 7, 8], "101": [5, 8], "878": 5, "144": 5, "t127": 5, "t135": 5, "91": [5, 8], "456": 5, "78": [5, 7, 8], "59": [5, 8], "t140": 5, "326": 5, "t158": 5, "204": 5, "350": 5, "002": [5, 7], "nuncondit": 5, "uncondit": 5, "206": 5, "440": 5, "156": 5, "t633": 5, "t670": 5, "226": 5, "45": 5, "nconting": 5, "accrual": 5, "nconcentr": 5, "attribut": [5, 6, 7, 8, 9], "46": 5, "t67": 5, "098": 5, "082": 5, "062": 5, "569": 5, "895": 5, "458": 5, "207": 5, "nonrecur": 5, "t142": 5, "196": 5, "t138": 5, "t147": 5, "859": 5, "nchina": 5, "n66": 5, "t181": 5, "887": 5, "t172": 5, "269": 5, "nlong": 5, "664": 5, "797": 5, "778": 5, "219": 5, "nopinion": 5, "nwe": 5, "fairli": 5, "pcaob": 5, "sponsor": 5, "treadwai": 5, "2013": 5, "unqualifi": [5, 6], "thereon": 5, "nthese": 5, "misstat": 5, "fraud": [5, 8], "ndescript": 5, "naudit": 5, "nhow": 5, "nmatter": 5, "qualifi": 5, "letter": [5, 6], "advisor": 5, "ernst": 5, "llp": 5, "auditor": [5, 6], "2009": 5, "nsan": 5, "jose": 5, "nnovemb": 5, "coso": 5, "nour": 5, "ndefinit": 5, "disposit": 5, "receipt": 5, "nevalu": 5, "nbase": 5, "supervis": [5, 7, 8, 9], "13a": 5, "15d": 5, "ninher": 5, "paragraph": 5, "51": [5, 8, 9], "ninsid": 5, "deirdr": 5, "brien": 5, "vice": 5, "presid": 5, "affirm": 5, "april": 5, "withhold": 5, "remitt": 5, "mr": 5, "copi": [5, 6], "solicit": 5, "00042": 5, "nincorpor": 5, "texhibit": 5, "descript": [5, 6, 7, 8, 9], "tform": 5, "tfile": 5, "nrestat": 5, "namend": 5, "bylaw": 5, "nindentur": 5, "york": [5, 6, 7, 9], "mellon": 5, "truste": 5, "noffic": 5, "certif": 5, "2018": 5, "85": [5, 7, 8], "05": [5, 6], "2044": 5, "februari": 5, "2045": 5, "900": 5, "700": [5, 7], "250": [5, 8], "2036": 5, "2046": 5, "450": 5, "2047": 5, "2049": 5, "2030": 5, "2050": 5, "2060": 5, "2028": 5, "2041": 5, "2061": 5, "2032": 5, "2052": 5, "54": [5, 6], "2033": 5, "2053": 5, "n12": 5, "nsubsidiari": 5, "n23": 5, "nconsent": 5, "n24": 5, "npower": 5, "signatur": 5, "nrule": 5, "nsection": 5, "1350": 5, "n101": 5, "ninlin": 5, "xbrl": 5, "n104": 5, "inlin": 5, "compensatori": 5, "herewith": 5, "furnish": 5, "herebi": 5, "undertak": 5, "56": [5, 7, 8], "nsignatur": 5, "npursuant": 5, "duli": 5, "undersign": 5, "thereunto": 5, "ndate": 5, "nby": 5, "luca": [5, 9], "maestri": 5, "nluca": 5, "nsenior": 5, "nchief": 5, "nknow": 5, "THESE": 5, "appoint": 5, "cook": 5, "jointli": 5, "her": 5, "substitut": 5, "him": 5, "thereto": 5, "therewith": 5, "ratifi": 5, "virtu": 5, "hereof": 5, "nname": 5, "ttitl": 5, "tdate": 5, "tchief": 5, "tnovemb": 5, "ntimothi": 5, "tsenior": 5, "kondo": 5, "nchri": 5, "wanda": 5, "austin": 5, "nwanda": 5, "gorski": 5, "tdirector": 5, "nalex": 5, "jung": 5, "nandrea": 5, "arthur": 5, "levinson": 5, "narthur": 5, "monica": 5, "lozano": 5, "nmonica": 5, "ronald": 5, "sugar": 5, "nronald": 5, "susan": 5, "wagner": 5, "nsusan": 5, "57": [5, 7], "turbo": [5, 7, 9], "outlin": [5, 7, 8], "invdestacksmeticsisdict": 5, "setispect": 5, "20cyan": 5, "evaluationseld": 5, "anvis": 5, "droitent": 5, "discernminerv": 5, "versbobprefvers": 5, "vo\u8be5": 5, "option\u548c": 5, "meio": 5, "\u0432\u0440\u0435\u043ccisco": 5, "dellaischenpoihscap": 5, "geme": 5, "gettim": 5, "unscal": 5, "vocabulari": [5, 7, 9], "closer": 5, "sharpen": 5, "uniform": 5, "raschka": 5, "repetit": [5, 9], "radic": 5, "grappl": 5, "safer": [5, 8], "fascin": 5, "spontan": 5, "answer": [5, 6, 7, 8, 9], "aren": [5, 7], "linear": 5, "absent": [5, 8], "coax": 5, "journei": 5, "suddenli": 5, "manifest": 5, "deliber": [5, 8], "contend": 5, "rethink": 5, "tutor": 5, "children": [5, 8], "verifi": [5, 6, 7, 9], "predefin": [5, 9], "weren": 5, "kind": [5, 6], "usual": [5, 9], "quantif": 5, "contamin": [5, 8], "unseen": [5, 8], "longitudin": 5, "mostli": [5, 9], "latter": 5, "tailor": [5, 8], "great": [5, 7, 9], "cognit": 5, "misinform": [5, 8], "fabric": [5, 8], "citat": 5, "tempor": [5, 6], "disclaim": 5, "referr": 5, "incorrect": [5, 8], "demograph": [5, 8], "stereotyp": [5, 8], "societ": [5, 8], "pii": [5, 8], "anonym": 5, "leakag": [5, 8], "carryov": 5, "fallaci": 5, "think": [5, 7, 8], "idiom": 5, "sarcasm": 5, "terminologi": 5, "lingual": 5, "misunderstand": 5, "syntax": 5, "scan": [5, 6], "compat": [5, 6, 7, 9], "overconfid": [5, 6], "clariti": [5, 6, 8, 9], "audienc": 5, "densiti": 5, "satisfact": [5, 9], "misus": [5, 8], "moral": 5, "co2": 5, "etc": [5, 6, 9], "palm": [5, 7], "easi": [5, 6, 7, 8], "synthet": [5, 7, 8, 9], "templat": [5, 6, 9], "timeout": 5, "inter": 5, "rater": 5, "ti": 5, "holist": [5, 8], "built": [5, 7, 8, 9], "experiment": [5, 7, 9], "vi": 5, "categor": [5, 7, 8, 9], "intrins": [5, 7], "extrins": 5, "sequenc": [5, 6, 7, 9], "perplex": [5, 7], "downstream": [5, 9], "synthesi": 5, "discret": 5, "prefix": [5, 8], "roug": 5, "bleu": 5, "bilingu": 5, "understudi": 5, "overlap": [5, 6], "favor": [5, 7, 9], "breviti": 5, "insensit": 5, "semant": [5, 6, 9], "orient": [5, 8], "gist": 5, "meteor": 5, "synonym": 5, "paraphras": 5, "alongsid": [5, 8], "computation": [5, 6], "cider": 5, "consensu": 5, "tf": 5, "idf": 5, "caption": 5, "reliant": [5, 6], "corpu": [5, 6, 7], "ter": 5, "edit": [5, 8], "hypothesi": 5, "penal": 5, "bertscor": 5, "contextu": [5, 8], "bert": 5, "spice": 5, "proposit": [5, 7], "scene": [5, 6, 8], "analyst": [5, 6], "rouge_1": 5, "rouge_2": 5, "ideal": [5, 7, 8, 9], "setup": [5, 7, 8, 9], "evaluate_summari": 5, "unigram": 5, "bigram": 5, "absl": 5, "py": [5, 9], "rouge_scor": 5, "generated_summari": 5, "reference_summari": 5, "google_bleu": 5, "bleu_scor": 5, "rouge1": 5, "rouge2": 5, "arbitrari": 5, "chosen": [5, 8], "sentence1": 5, "cat": [5, 8], "sat": 5, "mat": 5, "sentence2": 5, "ate": 5, "3333333333333333": 5, "7272727272727272": 5, "4444444444444445": 5, "generate_summari": 5, "summir": 5, "liner": 5, "evaluate_summary_model": 5, "model_benchmark": 5, "models_test": 5, "benchmark_summari": 5, "model_summari": 5, "evaluation_result": 5, "statu": 5, "concis": [5, 7], "element": [5, 8, 9], "verbos": [5, 7, 8, 9], "peripher": 5, "quit": [5, 6, 7, 9], "convei": 5, "breadth": 5, "Of": [5, 7, 8], "vibe": 5, "visualize_prompt_comparison": 5, "matplotlib": 5, "radar": 5, "radar_plot": 5, "tmp": 5, "ipykernel_1652501": 5, "940173201": 5, "userwarn": [5, 9], "figurecanvasagg": 5, "largest": [5, 7], "sarmah": 5, "granular": [5, 6, 7], "likert": 5, "ensembl": 5, "repeatedli": [5, 6], "fluenci": 5, "refin": 5, "integ": [5, 9], "rubric": 5, "hollist": 5, "judgeevalu": 5, "grammar": [5, 7, 9], "evaluate_with_llm": 5, "criterion": 5, "judge_model": 5, "candidate_summari": 5, "grammat": 5, "y": [5, 6, 8, 9], "z": 5, "w": [5, 6, 7, 8], "benchmark_model": 5, "test_model": 5, "input_text": [5, 6, 7], "trillion": [5, 7, 9], "evals_list": 5, "1775618912": 5, "slightli": 5, "drift": [5, 8], "lowest": [5, 7], "firstli": 5, "overhead": [5, 7], "egocentr": 5, "tight": 5, "medicin": [5, 8], "glider": 5, "deshpand": 5, "3b": 5, "685": 5, "aplic": 5, "golden": 5, "earlier": [5, 8], "depict": [5, 8, 9], "multilingu": [5, 7, 8], "arena": 5, "randomli": 5, "customiz": [5, 7, 8], "irrelev": 5, "unhelp": [5, 8], "occasion": 5, "rare": 5, "perfectli": 5, "cater": [5, 7], "critiqu": [5, 8], "elo": 5, "exam": 5, "probe": [5, 8], "certifi": 5, "glue": 5, "entail": [5, 7], "superglu": 5, "successor": 5, "grew": 5, "big": [5, 7], "bench": [5, 7], "srivastava": 5, "truthfulqa": [5, 7], "multitask": 5, "hendryck": [5, 8], "multidisciplinari": 5, "stanford": 5, "helm": 5, "multidimension": 5, "surround": [5, 7, 8, 9], "humanev": [5, 7], "lmsy": 5, "brought": 5, "dialogu": [5, 7], "chiang": 5, "gather": 5, "hundr": [5, 7], "alpacaev": 5, "duboi": 5, "mt": 5, "argilla": 5, "mila": 5, "mit": [5, 7], "contributor": [5, 7, 9], "western": 5, "centric": 5, "divid": [5, 6, 8], "subset": [5, 8], "agnost": 5, "dialect": 5, "render": [5, 8], "crowdsourc": 5, "livebench": 5, "white": [5, 8], "resili": [5, 6, 8], "meaningfulli": 5, "satur": 5, "zebralog": 5, "grid": 5, "puzzl": 5, "brailsford": 5, "1999": 5, "lsat": 5, "hous": 5, "clue": 5, "deduct": 5, "programmat": [5, 9], "2x2": 5, "6x6": 5, "shot": [5, 8, 9], "reductio": 5, "ad": [5, 6, 7, 9], "absurdum": 5, "hard": [5, 6], "10b": 5, "counterfactu": 5, "mileston": [5, 7], "came": 5, "arc": 5, "prize": [5, 8], "chollet": 5, "mike": [5, 6, 8], "knoop": 5, "founder": 5, "zapier": 5, "fran\u00e7oi": 5, "creator": [5, 7], "kera": 5, "genuin": 5, "agi": 5, "possess": [5, 6], "elementari": 5, "novelti": 5, "interpol": 5, "synthes": 5, "fly": 5, "brute": 5, "pixel": 5, "unbeaten": 5, "win": [5, 7], "takeawai": 5, "vertic": [5, 8], "finbench": 5, "legalbench": 5, "guha": 5, "berkelei": 5, "bfcl": 5, "patil": 5, "fourrier": 5, "bespok": 5, "sdk": 5, "autoregress": 5, "sub": [5, 7], "liter": 5, "disturb": 5, "zero": [5, 7, 8, 9], "varianc": [5, 8], "yt": 5, "ut": 5, "ol": 5, "heteroscedast": 5, "regress": 5, "bivari": 5, "evaluationtrack": 5, "pipelineparamet": 5, "cache_dir": 5, "max_sampl": 5, "basemodelconfig": 5, "evaluation_track": 5, "model_config": 5, "parallelismmanag": 5, "envconfig": 5, "is_accelerate_avail": 5, "datetim": [5, 6], "timedelta": [5, 6], "initprocessgroupkwarg": 5, "create_evaluation_pipelin": 5, "float16": 5, "kwargs_handl": 5, "3000": 5, "save_detail": 5, "pipeline_param": 5, "launcher_typ": 5, "env_config": 5, "override_batch_s": 5, "use_chat_templ": 5, "trust_remote_cod": 5, "pipeline_paramet": 5, "schemat": [5, 6], "vllm": [5, 9], "tgi": 5, "num_few_shot": 5, "bar": 5, "bigbench": 5, "winogrand": 5, "hellaswag": 5, "nlp": [5, 6, 7, 8], "save_and_push_result": 5, "show_result": 5, "model_arg": 5, "send": [5, 6, 7, 8, 9], "serverless": 5, "inference_server_address": 5, "inference_server_auth": 5, "model_id": 5, "null": 5, "bash": [5, 7], "command": [5, 6, 7], "model_config_path": 5, "endpoint_model": 5, "llama3": 5, "qwen2": [5, 7, 9], "alibaba": [5, 7, 9], "5b": [5, 7, 9], "hui": [5, 7], "allal": [5, 7], "cluster": 5, "noteworthi": [5, 7], "grain": [5, 7, 9], "salt": [5, 9], "modular": 5, "offici": 5, "revisit": 5, "langchain": [5, 6], "trace": 5, "langchain_tracing_v2": 5, "langchain_api_kei": 5, "hf_evalu": 5, "langsmith_evalu": 5, "ls_client": 5, "dataset_nam": 5, "create_dataset": 5, "create_exampl": 5, "dataset_id": 5, "calculate_scor": 5, "reference_output": 5, "oai_client": 5, "xp_model_nam": 5, "lastli": 5, "run_evalu": 5, "And": [5, 6, 7, 8], "upload_result": 5, "experiment_prefix": 5, "num_repetit": 5, "386a3620": 5, "9e1cc3cb": 5, "9d6a": 5, "4356": 5, "ab34": 5, "138e0abe8be4": 5, "8741976e": 5, "5268": 5, "4b75": 5, "949f": 5, "99477dde5d64": 5, "selectedsess": 5, "b831dc1e": 5, "90bc": 5, "4ed8": 5, "8080": [5, 7], "fb42444724d6": 5, "4it": 5, "latest": [5, 6, 7, 8, 9], "tobia": [5, 9], "evaluate_modul": 5, "6fc70b7be0088120a372dfdd5d320b39b8bb3630cb8029b193941d9376e86bb0": 5, "tue": 5, "nov": [5, 7], "couldn": 5, "5it": 5, "5053784e": 5, "64445871": 5, "a53c": 5, "44b1": 5, "a422": 5, "4f49b2f9656f": 5, "69": [5, 8], "4b29f3c9": 5, "9ef7e39a": 5, "2add": 5, "410c": 5, "89f8": 5, "9f1a8b198cf1": 5, "61": [5, 8], "insert": [5, 6], "combined_df": 5, "concat": [5, 8], "ignore_index": [5, 8], "execution_tim": 5, "example_id": 5, "333333": 5, "224388": 5, "feb10f92": 5, "3167": 5, "41f3": 5, "bb1c": 5, "d271153a31a8": 5, "5b196b22": 5, "9f4c": 5, "489c": 5, "b020": 5, "7823208b42d6": 5, "348101": 5, "722464": 5, "c310f159": 5, "064a": 5, "4035": 5, "97c3": 5, "a25bbf43abc2": 5, "386076": 5, "704104": 5, "f7f24899": 5, "dd50": 5, "409e": 5, "93cc": 5, "6fb1622b60bf": 5, "443038": 5, "725059": 5, "242856d6": 5, "efb5": 5, "4101": 5, "b1cf": 5, "5805532838ac": 5, "373418": 5, "795302": 5, "ce975169": 5, "a0ab": 5, "40ce": 5, "8e32": 5, "efa28d06079d": 5, "stat": [5, 7], "groupbi": [5, 8], "agg": [5, 8], "sort": 5, "sort_valu": 5, "subplot": 5, "pyplot": 5, "plt": 5, "ax1": 5, "ax2": 5, "figsiz": 5, "2ecc71": 5, "3498db": 5, "e74c3c": 5, "bleu_mean": 5, "bleu_std": 5, "enumer": [5, 6, 8], "errorbar": 5, "yerr": 5, "fmt": 5, "markers": 5, "capsiz": 5, "set_ylabel": 5, "set_titl": 5, "set_xtick": 5, "set_xticklabel": 5, "rotat": 5, "set_ylim": 5, "bottom": 5, "legend": 5, "exec_mean": 5, "exec_std": 5, "tight_layout": 5, "ndetail": 5, "4038": 5, "0453": 5, "7815": 5, "0433": 5, "3768": 5, "0424": 5, "8343": 5, "2208": 5, "3519": 5, "0775": 5, "9122": 5, "1482": 5, "377": 5, "042": 5, "078": 5, "slower": [5, 8], "04": [5, 7], "interestingli": 5, "decoupl": 5, "reload": 5, "facilit": [5, 8], "promptfooconfig": 5, "model_comparison": 5, "pretti": [5, 8], "dump": 5, "default_flow_styl": 5, "sort_kei": 5, "prompt1": 5, "defaulttest": 5, "1000m": 5, "eval_data": 5, "latency_m": 5, "totallatencym": 5, "token_usag": 5, "tokenusag": 5, "assert_pass": 5, "assertpasscount": 5, "assert_fail": 5, "assertfailcount": 5, "prompt_token": [5, 7], "num_request": 5, "numrequest": 5, "2463": 5, "000035": 5, "3773": 5, "004620": 5, "1669": 5, "000091": 5, "1669m": 5, "highest": [5, 7, 9], "3773m": 5, "00462": 5, "promptfool": 5, "manual": [5, 6, 7, 8], "redefin": 5, "prompt_comparison": 5, "prompt2": 5, "prompt3": 5, "prompt_fil": 5, "prompt_cont": 5, "BE": 5, "again": 5, "prompt_id": 5, "promptid": 5, "gradingresult": 5, "df_raw": 5, "reset_index": [5, 8], "poorli": 5, "eas": [5, 7, 8, 9], "hf": [5, 7], "plain": [5, 6, 7], "vanilla": 5, "defi": 5, "accustom": 5, "legaci": 5, "unsustain": 5, "prd": 5, "cultiv": [5, 8], "organiz": 5, "alb": [5, 7], "loubna": [5, 7], "anton": [5, 7], "lozhkov": [5, 7], "bakouch": [5, 7], "gabriel": [5, 7, 8], "mart\u00edn": [5, 7, 8], "bl\u00e1zquez": [5, 7], "lewi": [5, 6, 7], "tunstal": [5, 7], "agust\u00edn": [5, 7], "piquer": [5, 7], "andr": [5, 6, 7], "marafioti": [5, 7], "cyril": [5, 7], "zakka": [5, 7], "leandro": [5, 7], "werra": [5, 7], "wolf": [5, 7], "are24": 5, "judgearena": 5, "bps99": 5, "salli": 5, "pott": 5, "barbara": 5, "557": [5, 8], "sciencedirect": 5, "s0377221798003646": 5, "doi": [5, 6, 8, 9], "1016": 5, "s0377": 5, "2217": 5, "00364": 5, "ctj": 5, "jerri": [5, 8], "tworek": [5, 8], "heewoo": [5, 8], "jun": [5, 8], "qime": [5, 8], "henriqu": [5, 8], "pond": [5, 8], "de": [5, 8], "oliveira": [5, 8], "pinto": [5, 8], "harri": [5, 8], "yuri": 5, "burda": 5, "greg": [5, 8], "brockman": [5, 8], "raul": [5, 8], "puri": [5, 8], "gretchen": [5, 8], "krueger": [5, 8], "petrov": [5, 8], "heidi": 5, "khlaaf": 5, "girish": [5, 8], "sastri": [5, 8], "brook": [5, 8], "chan": [5, 8], "grai": [5, 8], "ryder": [5, 8], "mikhail": [5, 8], "pavlov": [5, 8], "alethea": [5, 8], "lukasz": 5, "kaiser": [5, 8], "mohammad": [5, 8], "bavarian": [5, 8], "clemen": [5, 8], "winter": [5, 8], "philipp": 5, "tillet": [5, 8], "felip": [5, 8], "petroski": [5, 8], "dave": [5, 8], "cum": [5, 8], "plappert": 5, "fotio": 5, "chantzi": [5, 8], "barn": 5, "ariel": 5, "herbert": 5, "voss": [5, 8], "hebgen": 5, "guss": 5, "nichol": 5, "paino": [5, 8], "nikola": [5, 8], "tezak": [5, 8], "babuschkin": [5, 8], "suchir": [5, 8], "balaji": [5, 8], "shantanu": [5, 8], "jain": [5, 8], "hess": [5, 8], "carr": 5, "josh": [5, 8], "achiam": [5, 8], "vedant": 5, "misra": 5, "evan": [5, 7, 8], "morikawa": [5, 8], "matthew": 5, "knight": [5, 8], "mile": [5, 8], "brundag": [5, 8], "mira": [5, 8], "murati": [5, 8], "kati": [5, 8], "mayer": [5, 8], "bob": [5, 8, 9], "mcgrew": [5, 8], "ilya": [5, 8], "sutskev": [5, 8], "wojciech": [5, 8], "zaremba": [5, 8], "2107": 5, "03374": 5, "cz": 5, "lianmin": 5, "ying": 5, "sheng": 5, "anastasio": 5, "angelopoulo": 5, "tianl": 5, "dacheng": 5, "banghua": 5, "jordan": [5, 8], "gonzalez": 5, "ion": 5, "stoica": 5, "04132": 5, "cho24a": 5, "francoi": 5, "arcpriz": 5, "cho24b": 5, "drcw": 5, "darshan": 5, "selvan": 5, "sunitha": 5, "ravi": 5, "sky": 5, "ch": 5, "bartosz": 5, "mielczarek": 5, "anand": [5, 8], "kannappan": [5, 8], "qian": [5, 8], "14140": 5, "dglh24": 5, "yann": 5, "bal\u00e1z": 5, "galambosi": 5, "tatsunori": 5, "hashimoto": 5, "debia": 5, "04475": 5, "fac24a": 5, "wiki": [5, 9], "fac24b": 5, "fac24c": 5, "model_doc": 5, "fac24d": 5, "cookbook": 5, "llm_judg": 5, "fac24f": 5, "fhwt23": 5, "cl\u00e9mentin": 5, "nathan": 5, "habib": 5, "gnh": 5, "julian": 5, "nyarko": 5, "ho": 5, "r\u00e9": 5, "adam": [5, 8], "chilton": 5, "aditya": [5, 8], "narayana": 5, "chohla": 5, "brandon": [5, 8, 9], "waldon": 5, "rockmor": 5, "diego": 5, "zambrano": 5, "dmitri": 5, "talisman": 5, "enam": 5, "hoqu": 5, "faiz": 5, "surani": 5, "frank": [5, 8], "fagan": 5, "galit": 5, "sarfati": 5, "gregori": 5, "dickinson": 5, "haggai": 5, "porat": 5, "hegland": 5, "jessica": [5, 8], "joe": [5, 8], "nudel": 5, "joel": [5, 8], "niklau": 5, "nai": 5, "choi": 5, "margaret": [5, 7], "hagan": 5, "megan": 5, "livermor": 5, "nikon": 5, "rasumov": 5, "rahe": 5, "nil": 5, "holzenberg": 5, "noam": 5, "kolt": 5, "henderson": 5, "rehaag": 5, "sharad": 5, "shang": 5, "spencer": 5, "sunni": 5, "gandhi": 5, "zur": 5, "varun": 5, "iyer": 5, "zehua": 5, "2308": 5, "11462": 5, "hbb": 5, "collin": 5, "burn": 5, "steven": [5, 8], "basart": [5, 8], "zou": [5, 8], "manta": [5, 8], "mazeika": [5, 8], "03300": 5, "hbd": 5, "maxwel": 5, "forb": 5, "yejin": 5, "curiou": 5, "neural": [5, 9], "degener": 5, "1904": 5, "09751": 5, "hyc": [5, 7], "binyuan": [5, 7], "zeyu": [5, 7], "cui": [5, 7], "jiaxi": [5, 7], "dayiheng": [5, 7], "tianyu": [5, 7], "jiajun": [5, 7], "kai": [5, 7, 8], "dang": [5, 7], "coder": [5, 7], "preprint": [5, 7, 9], "2409": [5, 7, 8], "12186": [5, 7], "lx": 5, "zhen": 5, "xiaohan": 5, "jia": 5, "yuxuan": 5, "lai": 5, "chongyang": 5, "shuai": 5, "nlg": 5, "07103": 5, "lbl": 5, "bommasani": 5, "toni": 5, "dimitri": 5, "tsipra": 5, "dilara": 5, "soylu": 5, "michihiro": 5, "yasunaga": 5, "yian": 5, "deepak": 5, "narayanan": 5, "yuhuai": 5, "newman": 5, "binhang": 5, "bobbi": 5, "ce": 5, "christian": [5, 8], "cosgrov": 5, "acosta": 5, "nava": [5, 8], "drew": 5, "hudson": 5, "zelikman": 5, "esin": 5, "durmu": 5, "faisal": 5, "ladhak": 5, "frieda": 5, "rong": [5, 6], "ren": [5, 7], "huaxiu": 5, "yao": [5, 8, 9], "jue": 5, "keshav": 5, "santhanam": 5, "laurel": 5, "lucia": 5, "mert": 5, "yuksekgonul": 5, "mirac": 5, "suzgun": 5, "niladri": 5, "chatterji": 5, "omar": 5, "khattab": 5, "chi": [5, 9], "sang": 5, "shibani": [5, 8], "santurkar": [5, 8], "surya": 5, "icard": 5, "tianyi": 5, "vishrav": 5, "chaudhari": 5, "xuechen": 5, "yuhui": 5, "yuta": 5, "koreeda": 5, "2211": 5, "09110": 5, "lbc24": 5, "ronan": 5, "bra": 5, "allenai": 5, "lhe22": [5, 7, 8], "stephani": [5, 7, 8], "owain": [5, 7, 8], "mimic": [5, 7, 8], "falsehood": [5, 7, 8], "2109": [5, 7, 8], "07958": [5, 7, 8], "pzwg23": 5, "shishir": 5, "tianjun": 5, "xin": [5, 8], "gorilla": 5, "15334": 5, "pro24": 5, "dev": 5, "ras24": 5, "sebastian": [5, 6], "scratch": 5, "1633437166": 5, "sll": 5, "bhaskarjit": 5, "mingshu": 5, "jingrao": 5, "lyu": 5, "nathalia": 5, "castellano": 5, "pasquali": 5, "dhagash": 5, "12148": 5, "srf": 5, "shivalika": 5, "angelika": 5, "roman": [5, 8], "adelani": 5, "ngui": 5, "vila": 5, "suero": 5, "peerat": 5, "limkonchotiwat": 5, "kelli": 5, "marchisio": 5, "qi": 5, "leong": 5, "yosephin": 5, "susanto": 5, "raymond": [5, 8], "ng": [5, 8], "shayn": 5, "longpr": 5, "ko": 5, "madelin": 5, "antoin": 5, "bosselut": 5, "oh": 5, "leshem": 5, "choshen": 5, "daphn": 5, "ippolito": 5, "enzo": [5, 9], "ferrant": 5, "marzieh": 5, "fadae": 5, "beyza": 5, "ermi": 5, "sara": 5, "hooker": 5, "linguist": [5, 6, 8], "03304": 5, "srr": 5, "aarohi": 5, "abhinav": 5, "rastogi": 5, "abhishek": 5, "rao": 5, "abu": 5, "awal": 5, "shoeb": 5, "abubakar": 5, "abid": [5, 7], "fisch": 5, "santoro": 5, "gupta": 5, "adri\u00e0": 5, "garriga": 5, "alonso": 5, "agnieszka": 5, "kluska": 5, "aitor": 5, "lewkowycz": 5, "akshat": 5, "warstadt": 5, "alexand": [5, 8, 9], "kocurek": 5, "ali": [5, 8], "safaya": 5, "tazarv": 5, "aman": 5, "hussain": 5, "dsouza": 5, "ambros": 5, "slone": 5, "ameet": 5, "rahan": 5, "anantharaman": 5, "ander": 5, "andreassen": 5, "madotto": 5, "santilli": 5, "stuhlm\u00fcller": 5, "la": 5, "lampinen": 5, "angelica": 5, "anh": 5, "vuong": 5, "animesh": 5, "gottardi": 5, "antonio": 5, "norelli": 5, "anu": 5, "venkatesh": 5, "arash": 5, "gholamidavoodi": 5, "arfa": 5, "tabassum": 5, "arul": 5, "menez": 5, "arun": [5, 8], "kirubarajan": 5, "asher": 5, "mullokandov": 5, "ashish": 5, "sabharw": 5, "herrick": 5, "avia": 5, "efrat": 5, "aykut": 5, "erdem": 5, "ayla": 5, "karaka\u015f": 5, "bao": [5, 7, 8], "loe": 5, "barret": [5, 8], "zoph": [5, 8], "bart\u0142omiej": 5, "bojanowski": 5, "batuhan": 5, "\u00f6zyurt": 5, "behnam": 5, "hedayatnia": 5, "neyshabur": 5, "inden": 5, "benno": 5, "stein": 5, "berk": 5, "ekmekci": 5, "blake": 5, "howald": 5, "bryan": 5, "orinion": 5, "diao": 5, "dour": 5, "stinson": 5, "cedrick": 5, "argueta": 5, "c\u00e9sar": 5, "ferri": 5, "ram\u00edrez": 5, "chandan": 5, "charl": 5, "rathkopf": 5, "chenlin": 5, "meng": 5, "chitta": 5, "baral": 5, "chiyu": 5, "callison": 5, "burch": 5, "voigt": 5, "cindi": 5, "ramirez": 5, "clara": 5, "rivera": 5, "clemencia": 5, "siro": 5, "colin": [5, 7], "raffel": [5, 7], "courtnei": 5, "ashcraft": 5, "cristina": 5, "garbacea": 5, "damien": [5, 8], "sileo": 5, "garrett": 5, "kilman": 5, "freeman": 5, "khashabi": 5, "levi": [5, 8], "mosegu\u00ed": 5, "gonz\u00e1lez": 5, "perszyk": 5, "danqi": 5, "dar": 5, "gilboa": 5, "dohan": [5, 8], "drakard": 5, "jurgen": 5, "debajyoti": 5, "datta": 5, "deni": 5, "emelin": 5, "kleyko": 5, "deniz": 5, "yuret": 5, "derek": [5, 8], "tam": [5, 9], "dieuwk": 5, "hupk": 5, "diganta": 5, "dilyar": 5, "buzan": 5, "coelho": 5, "mollo": 5, "diyi": 5, "dylan": 5, "schrader": 5, "ekaterina": 5, "shutova": 5, "ekin": 5, "dogu": 5, "cubuk": 5, "elad": 5, "segal": 5, "eleanor": 5, "hagerman": 5, "donowai": 5, "elli": 5, "pavlick": 5, "rodola": 5, "emma": 5, "lam": 5, "chu": [5, 8], "erkut": 5, "erni": 5, "dyer": 5, "jerzak": 5, "eunic": 5, "engefu": 5, "manyasi": 5, "evgenii": 5, "zheltonozhskii": 5, "fanyu": 5, "fatemeh": 5, "siar": 5, "fernando": 5, "mart\u00ednez": 5, "plume": 5, "francesca": 5, "happ\u00e9": 5, "gaurav": 5, "genta": 5, "indra": 5, "winata": 5, "gerard": 5, "melo": 5, "germ\u00e1n": 5, "kruszewski": 5, "giambattista": [5, 8], "parascandolo": [5, 8], "giorgio": 5, "mariani": 5, "gloria": 5, "gonzalo": 5, "jaimovitch": 5, "l\u00f3pez": 5, "gregor": 5, "betz": 5, "gui": [5, 7], "gur": 5, "hana": 5, "galijasev": 5, "rashkin": 5, "hannaneh": 5, "hajishirzi": 5, "harsh": 5, "hayden": 5, "bogar": 5, "henri": [5, 8], "shevlin": 5, "hinrich": 5, "sch\u00fctze": 5, "hiromu": 5, "yakura": 5, "hongm": 5, "hugh": 5, "mee": 5, "wong": [5, 6, 8], "isaac": 5, "nobl": 5, "jaap": 5, "jumelet": 5, "geissing": 5, "jaehoon": 5, "jaim": 5, "fern\u00e1ndez": 5, "fisac": 5, "simon": 5, "koppel": 5, "koco\u0144": 5, "jana": 5, "thompson": [5, 7, 8], "janel": 5, "wingfield": 5, "jarema": 5, "radom": 5, "jascha": 5, "sohl": [5, 8], "dickstein": 5, "phang": 5, "yosinski": 5, "jekaterina": 5, "novikova": 5, "jell": 5, "bosscher": 5, "jennif": 5, "marsh": 5, "jeroen": 5, "taal": 5, "engel": 5, "jesujoba": 5, "alabi": 5, "jiam": 5, "jillian": 5, "joan": 5, "waweru": 5, "burden": 5, "bali": 5, "batcheld": 5, "berant": 5, "j\u00f6rg": 5, "frohberg": 5, "jo": 5, "rozen": 5, "orallo": 5, "boudeman": 5, "guerr": 5, "tenenbaum": 5, "joyc": 5, "chua": 5, "kanclerz": 5, "karen": 5, "livescu": 5, "karl": 5, "krauth": 5, "karthik": 5, "gopalakrishnan": 5, "katerina": 5, "ignatyeva": 5, "katja": 5, "markert": 5, "kaustubh": 5, "dhole": 5, "gimpel": 5, "omondi": 5, "kori": 5, "mathewson": 5, "kristen": 5, "chiafullo": 5, "ksenia": 5, "shkaruta": 5, "shridhar": 5, "kyle": [5, 6, 8], "mcdonel": 5, "richardson": 5, "laria": 5, "reynold": 5, "leo": [5, 8], "dugan": 5, "lianhui": 5, "lidia": 5, "contrera": 5, "ochando": 5, "morenc": 5, "moschella": 5, "luci": 5, "ludwig": 5, "schmidt": [5, 8], "luheng": 5, "olivero": 5, "col\u00f3n": 5, "metz": [5, 8], "l\u00fctfi": 5, "kerem": 5, "\u015fenel": 5, "maarten": [5, 8], "bosma": 5, "sap": [5, 8], "maartj": 5, "hoev": 5, "maheen": 5, "farooqi": 5, "manaal": 5, "faruqui": 5, "marco": 5, "baturan": 5, "marelli": 5, "maru": 5, "maria": 5, "quintana": 5, "tolkiehn": 5, "mario": [5, 8], "giulianelli": 5, "martha": 5, "potthast": 5, "leavitt": 5, "hagen": 5, "m\u00e1ty\u00e1": 5, "schubert": 5, "medina": [5, 8], "orduna": 5, "baitemirova": 5, "melodi": 5, "arnaud": 5, "melvin": 5, "mcelrath": 5, "yee": 5, "cohen": 5, "ivanitskii": 5, "starritt": 5, "strube": 5, "micha\u0142": 5, "sw\u0119drowski": 5, "michel": [5, 8], "bevilacqua": 5, "mihir": 5, "kale": 5, "cain": 5, "mime": 5, "mitch": 5, "walker": 5, "mo": 5, "tiwari": 5, "mohit": 5, "bansal": 5, "moin": 5, "aminnaseri": 5, "mor": 5, "geva": 5, "mozhdeh": 5, "gheini": 5, "mukund": 5, "varma": 5, "nanyun": 5, "peng": [5, 8], "nayeon": 5, "neta": 5, "krakov": 5, "doiron": 5, "nicol": 5, "martinez": 5, "nikita": 5, "nangia": 5, "nikla": 5, "decker": 5, "muennighoff": 5, "nitish": [5, 8], "shirish": [5, 8], "keskar": [5, 8], "niveditha": 5, "constant": 5, "fiedel": 5, "nuan": 5, "wen": [5, 6], "oliv": [5, 8], "agha": 5, "elbaghdadi": 5, "omer": 5, "moreno": 5, "casar": 5, "parth": 5, "doshi": 5, "pascal": 5, "fung": 5, "pu": 5, "vicol": 5, "pegah": 5, "alipoormolabashi": 5, "peiyuan": 5, "eckerslei": 5, "phu": 5, "mon": 5, "htut": 5, "pinyu": 5, "hwang": 5, "piotr": 5, "mi\u0142kowski": 5, "piyush": 5, "pouya": 5, "pezeshkpour": 5, "priti": 5, "oli": 5, "qiaozhu": 5, "qing": 5, "qinlang": 5, "rabin": 5, "banjad": 5, "rachel": [5, 8], "etta": 5, "rudolph": 5, "raefer": 5, "rahel": 5, "haback": 5, "ramon": 5, "risco": 5, "rapha\u00ebl": 5, "milli\u00e8r": 5, "rhythm": 5, "garg": [5, 7], "rif": 5, "saurou": 5, "riku": 5, "arakawa": 5, "robb": 5, "raymaek": 5, "rohan": 5, "sikand": 5, "novak": 5, "sitelew": 5, "lebra": 5, "rosann": 5, "rowan": [5, 8], "ruslan": 5, "salakhutdinov": 5, "stoval": 5, "teehan": 5, "sahib": 5, "saif": 5, "sajant": 5, "dillav": 5, "shleifer": 5, "wiseman": 5, "gruetter": 5, "schoenholz": 5, "sanghyun": 5, "sanjeev": 5, "kwatra": 5, "sarik": 5, "ghazarian": 5, "sayan": 5, "casei": [5, 8], "bischoff": 5, "gehrmann": 5, "schuster": 5, "sepideh": 5, "sadeghi": 5, "shadi": 5, "hamdan": 5, "sharon": 5, "shashank": 5, "sherri": 5, "shi": 5, "shikhar": 5, "shima": 5, "asaadi": 5, "shubh": 5, "pachchigar": 5, "shubham": 5, "toshniw": 5, "shyam": [5, 8], "upadhyai": 5, "shyamolima": 5, "debnath": 5, "siamak": 5, "shakeri": 5, "thormey": 5, "melzi": 5, "siva": 5, "reddi": 5, "sneha": 5, "priscilla": 5, "makini": 5, "soo": 5, "hwan": 5, "toren": 5, "sriharsha": 5, "hatwar": 5, "stanisla": 5, "dehaen": 5, "stefan": 5, "divic": 5, "stella": 5, "biderman": 5, "stephen": 5, "prasad": 5, "piantadosi": 5, "stuart": [5, 8], "shieber": 5, "summer": [5, 8], "misherghi": 5, "svetlana": 5, "kiritchenko": 5, "swaroop": 5, "tal": 5, "linzen": 5, "tariq": 5, "tatsu": 5, "te": 5, "th\u00e9o": 5, "desbord": 5, "theodor": 5, "rothschild": 5, "phan": [5, 8], "tiberiu": 5, "nkinyili": 5, "timo": 5, "schick": 5, "timofei": 5, "kornev": 5, "titu": 5, "tunduni": 5, "gerstenberg": 5, "trenton": 5, "trishala": 5, "neeraj": 5, "tushar": 5, "khot": 5, "shultz": 5, "uri": 5, "shaham": 5, "vera": 5, "demberg": 5, "victoria": [5, 8], "nyamai": 5, "vika": 5, "raunak": 5, "vinai": 5, "ramasesh": 5, "udai": 5, "prabhu": 5, "vishakh": 5, "padmakumar": 5, "vivek": [5, 6], "srikumar": [5, 6], "fedu": [5, 8], "wout": 5, "vossen": 5, "xiaoyu": 5, "tong": [5, 8], "xinran": 5, "xinyi": 5, "yadollah": 5, "yaghoobzadeh": 5, "yair": 5, "lakretz": 5, "yangqiu": 5, "yasaman": 5, "bahri": 5, "yichi": 5, "yide": 5, "yifu": 5, "yonatan": 5, "belinkov": 5, "yufang": 5, "seid": 5, "zhuoy": 5, "zijian": 5, "ziji": 5, "zirui": 5, "ziyi": 5, "extrapol": 5, "2206": 5, "04615": 5, "wpn": 5, "yada": 5, "pruksachatkun": 5, "amanpreet": 5, "hill": 5, "stickier": 5, "wsm": 5, "1804": 5, "07461": 5, "wtb": 5, "tai": 5, "borgeaud": 5, "dani": 5, "yogatama": 5, "denni": [5, 8], "donald": 5, "metzler": 5, "ed": 5, "oriol": 5, "vinyal": 5, "dean": 5, "07682": 5, "wdr": 5, "doolei": 5, "manlei": 5, "arka": [5, 8], "pal": 5, "feuer": 5, "siddhartha": 5, "ravid": 5, "shwartz": [5, 8], "ziv": 5, "khalid": [5, 7], "saifullah": 5, "siddartha": 5, "naidu": 5, "chinmai": 5, "hegd": 5, "lecun": 5, "goldstein": 5, "willi": 5, "neiswang": 5, "micah": 5, "goldblum": 5, "19314": 5, "yyh": 5, "baosong": [5, 7], "chengpeng": 5, "chengyuan": [5, 7], "fei": [5, 7], "guant": 5, "haoran": [5, 7], "huan": [5, 7], "jialong": 5, "jialin": 5, "jianhong": [5, 7], "tu": [5, 7], "jianwei": [5, 7], "jianxin": [5, 7], "jin": [5, 6, 8], "jingren": [5, 7], "jinz": 5, "jinzheng": 5, "junyang": [5, 7], "keme": [5, 7], "keqin": [5, 7], "kexin": [5, 7], "mingfeng": [5, 7], "xue": [5, 7, 8], "ni": [5, 6], "pei": [5, 7], "ru": 5, "men": [5, 7], "ruiz": 5, "runji": [5, 7], "shiji": 5, "sinan": 5, "tianhang": 5, "wenbin": 5, "ge": 5, "xiaodong": 5, "deng": 5, "xiaohuan": 5, "xingzhang": [5, 7], "xinyu": [5, 8], "xipin": 5, "xuancheng": [5, 7], "yichang": [5, 7], "wan": [5, 7], "yunfei": 5, "yuqiong": [5, 7], "zhenru": [5, 7], "zhihao": 5, "10671": 5, "zcl24": 5, "zhihan": 5, "cao": 5, "lizi": 5, "openreview": [5, 6], "forum": [5, 6], "aegrf1uy0p": 5, "zc": 5, "siyuan": 5, "zhuang": [5, 8], "zhanghao": 5, "yonghao": 5, "zi": 5, "zhuohan": 5, "xing": [5, 8], "2306": 5, "05685": 5, "huggingface24": 5, "metaai24": 5, "doubl": 6, "steve": [6, 8], "lclm": 6, "simultan": [6, 7, 8], "cutoff": 6, "amayuela": 6, "tail": 6, "kotha": 6, "unifi": [6, 7, 9], "codebas": [6, 7], "ingest": 6, "preprocess": [6, 7, 9], "parser": [6, 9], "microsoft": [6, 7], "autogen": 6, "powerpoint": 6, "ocr": 6, "exif": 6, "metadata": [6, 7], "docker": [6, 7], "container": [6, 7], "xlsx": 6, "text_cont": 6, "ibm": [6, 7, 8], "docx": 6, "pptx": 6, "layout": 6, "llamaindex": 6, "document_convert": 6, "documentconvert": 6, "export_to_markdown": 6, "presenc": 6, "merril": 6, "lynch": 6, "cio": 6, "outlook": 6, "forecast_file_path": 6, "result_md": 6, "forecast_result_docl": 6, "levenshtein": 6, "distanc": 6, "sequencematch": 6, "difflib": 6, "longest": 6, "levenshtein_similar": 6, "text1": 6, "text2": 6, "max_len": 6, "simple_similar": 6, "ratio": [6, 7], "forecast_result_md": 6, "13985705461925346": 6, "17779960707269155": 6, "readabl": 6, "messi": 6, "2025e": 6, "compos": [6, 7, 8], "financial_vari": 6, "financial_forecast": 6, "econforecast": 6, "extract_prompt": 6, "base_prompt": [6, 9], "extract_from_doc": 6, "twice": 6, "md_financi": 6, "docling_financi": 6, "easier": [6, 7, 8, 9], "gdp": 6, "cpi": 6, "fed": 6, "df_md_forecast": 6, "df_docling_forecast": 6, "despit": [6, 7, 9], "underweight": 6, "neutral": [6, 8], "overweight": 6, "chart": 6, "asset_class_docl": 6, "asset_class_md": 6, "df_md": 6, "df_docl": 6, "true_valu": 6, "df_comparison": 6, "cap": 6, "exempt": 6, "markitdown_accuraci": 6, "docling_accuraci": 6, "93": [6, 7, 8], "unstructur": [6, 7, 9], "sector": 6, "convert_and_export_t": 6, "file_path": 6, "doc_convert": 6, "start_tim": [6, 8], "conv_r": 6, "table_df": 6, "export_to_datafram": 6, "end_tim": 6, "2f": 6, "usd": 6, "wtd": 6, "mtd": 6, "ytd": 6, "djia": 6, "926": 6, "amp": 6, "051": 6, "277": 6, "russel": [6, 8], "2000": 6, "msci": 6, "817": [6, 8], "eaf": 6, "319": 6, "107": 6, "01": [6, 7], "66": [6, 8], "92": 6, "municip": 6, "79": [6, 8], "slight": 6, "discretionari": 6, "yellow": 6, "estat": 6, "orang": 6, "stapl": 6, "constructor": 6, "md_llm": 6, "llm_client": 6, "llm_model": 6, "png": 6, "overview": [6, 9], "showcas": 6, "bond": 6, "crude": 6, "oil": 6, "sit": 6, "648": 6, "ounc": 6, "euro": 6, "tactic": 6, "bofa": 6, "circl": [6, 8], "fetch": 6, "reassembl": 6, "max_output_token": 6, "statement": [6, 8], "10k": 6, "diagram": [6, 8], "charactertextsplitt": 6, "tiktoken": [6, 8], "sequenti": 6, "newlin": 6, "cheap": 6, "speciali": 6, "nltk": 6, "spaci": 6, "hierarch": [6, 8], "talk": 6, "theme": [6, 7, 8], "splitter": 6, "surpass": 6, "get_chunk": 6, "chunk_siz": 6, "chunk_overlap": 6, "langchain_text_splitt": 6, "text_splitt": 6, "from_tiktoken_encod": 6, "split_text": 6, "persona": 6, "langchain_cor": [6, 9], "prompttempl": 6, "get_base_prompt_templ": 6, "from_templ": 6, "llmchain": 6, "output_pars": 6, "stroutputpars": 6, "langchain_commun": 6, "chat_model": 6, "chatlitellm": 6, "get_llm_chain": 6, "prompt_templ": [6, 9], "llm_chain": [6, 9], "api_key_label": 6, "upper": 6, "_api_kei": 6, "get_dynamic_prompt_templ": 6, "get_dynamic_prompt_param": 6, "prompt_param": 6, "part_idx": 6, "total_part": 6, "chat_context": 6, "param": 6, "dynamic_prompt_param": 6, "concaten": 6, "generate_report": 6, "input_cont": 6, "llm_model_nam": 6, "report_part": 6, "num_part": 6, "dinam": 6, "priovid": 6, "invok": [6, 9], "cummul": 6, "max_chunk_s": 6, "max_chunk_overlap": 6, "gemini": [6, 7], "apple_report": 6, "report_cont": 6, "report_lin": 6, "splitlin": 6, "total_lin": 6, "quarter_lin": 6, "top_port": 6, "bottom_port": 6, "uncov": [6, 8, 9], "delv": 6, "consol": 6, "fewer": [6, 7, 8], "reaction": 6, "breakdown": [6, 8], "disciplin": 6, "appar": [6, 8], "subhead": 6, "depth": [6, 8], "2m": [6, 7], "harvard": [6, 7], "enrol": 6, "gov": [6, 8], "1039": 6, "birth": [6, 8], "democraci": 6, "tuesdai": 6, "magna": 6, "carta": 6, "trudg": 6, "dens": 6, "conversation": 6, "knowledge_bas": 6, "add_knowledge_bas": 6, "add_cit": 6, "bool": [6, 8], "num_quest": 6, "input_memori": 6, "response_memori": 6, "urls_memori": 6, "extractor": 6, "cic": 6, "citabl": 6, "passag": [6, 8], "corpora": 6, "formatted_cont": 6, "reference_id": 6, "wrapper": [6, 9], "content_gener": 6, "user_instruct": 6, "llmbackend": 6, "cache_ttl": 6, "cachedcont": 6, "display_nam": 6, "due_knowledge_bas": 6, "system_instruct": 6, "compose_prompt": 6, "conversation_config": 6, "ttl": 6, "generativemodel": 6, "from_cached_cont": 6, "cached_cont": 6, "behind": [6, 8], "quiz_inst": 6, "professor": 6, "difficulti": [6, 8], "dataset": [6, 9], "syllabu": 6, "kennedi": 6, "inaugur": 6, "lincoln": 6, "gettysburg": 6, "liberti": 6, "mayflow": 6, "abraham": 6, "gutenberg": 6, "kb": 6, "epub": 6, "pg": 6, "gemini_duo": 6, "genai_duo": 6, "duo": 6, "usage_metadata": 6, "38470": 6, "anytim": 6, "shap": 6, "mckechni": 6, "study_refer": 6, "pg10000": 6, "65363": 6, "pg65363": 6, "quizz": 6, "problemat": [6, 8], "misinterpret": 6, "awp": 6, "alfonso": 6, "liangm": 6, "pan": [6, 8], "wenhu": 6, "lun": 6, "ku": 6, "editor": [6, 8], "acl": [6, 8], "6416": 6, "6432": 6, "bangkok": 6, "thailand": 6, "aclanthologi": [6, 8], "383": 6, "18653": [6, 8], "v1": [6, 7, 8], "ksr24": 6, "suha": 6, "springer": 6, "aditi": 6, "raghunathan": 6, "twelfth": 6, "vrhif2hsrm": 6, "lcd": 6, "jinhyuk": 6, "zhuyun": 6, "dheeru": 6, "dua": 6, "devendra": 6, "sachan": 6, "boratko": 6, "luan": 6, "s\u00e9bastien": 6, "arnold": 6, "vincent": 6, "perot": 6, "siddharth": 6, "dalmia": 6, "hexiang": 6, "panupong": 6, "pasupat": 6, "aida": 6, "amini": 6, "cole": 6, "riedel": 6, "iftekhar": 6, "naim": 6, "ming": [6, 8], "guu": 6, "subsum": 6, "sql": 6, "13121": 6, "lpp": 6, "aleksandra": 6, "piktu": 6, "fabio": [6, 8], "petroni": 6, "vladimir": 6, "karpukhin": 6, "heinrich": 6, "k\u00fcttler": 6, "tau": 6, "yih": 6, "rockt\u00e4schel": 6, "douw": 6, "kiela": 6, "2005": 6, "11401": 6, "nbgc24": 6, "shiyu": 6, "kepe": 6, "bi": 6, "jiafeng": 6, "guo": [6, 8], "xueqi": 6, "cheng": [6, 9], "11375": 6, "11388": 6, "675": 6, "tdw": 6, "jiejun": 6, "zhicheng": 6, "dou": 6, "mang": 6, "weipeng": 6, "ji": 6, "htmlrag": 6, "02959": 6, "zlj": 6, "jiaji": 6, "yun": [6, 9], "metacognit": 6, "1453": 6, "1463": 6, "ny": [6, 8, 9], "usa": [6, 8, 9], "machineri": [6, 9], "1145": [6, 8, 9], "3589334": 6, "3645481": 6, "anthropic4a": 6, "langchain24": 6, "how_to": 6, "merrilllynch24": 6, "weekli": 6, "olui2": 6, "gwmol": 6, "di": 7, "hunter": 7, "photo": 7, "email": 7, "hipaa": 7, "properti": [7, 8], "gdpr": 7, "strict": [7, 8, 9], "iot": 7, "unreli": 7, "impract": 7, "slm": 7, "viabl": 7, "sensor": 7, "interconnect": 7, "frontend": 7, "garner": 7, "traction": 7, "yourself": 7, "aw": [7, 8], "bedrock": 7, "sambanova": 7, "sla": 7, "veloc": 7, "roadmap": 7, "commodit": 7, "winner": 7, "loser": 7, "condens": 7, "clean": 7, "2024t": 7, "versatil": 7, "72b": 7, "med": 7, "bloomberggpt": 7, "underw": 7, "adept": 7, "toxigen": 7, "alnajjar": 7, "13b": [7, 8], "outperform": 7, "32b": 7, "feasibl": 7, "modal": 7, "diagnosi": 7, "patient": 7, "necessit": 7, "deepseek": 7, "flagship": 7, "405b": 7, "pack": 7, "v3": [7, 8], "671": 7, "moe": 7, "mixtur": 7, "3x": [7, 8], "v2": [7, 8], "fraction": 7, "domin": 7, "cautiou": 7, "cautious": 7, "isol": [7, 8], "cpot": 7, "cpit": 7, "tco": 7, "tpot": 7, "ttft": 7, "gpqa": 7, "median": 7, "afford": 7, "lite": 7, "micro": 7, "encod": [7, 8, 9], "cent": 7, "1m": 7, "cheapest": 7, "phi": 7, "half": [7, 8], "permiss": [7, 8], "apach": 7, "simpler": [7, 9], "700m": 7, "100m": 7, "gemma": [7, 9], "grown": 7, "withdraw": 7, "incomplet": [7, 8], "unclear": 7, "15t": 7, "8t": 7, "fineweb": 7, "penedo": 7, "96": [7, 8], "crawl": 7, "snapshot": 7, "ablat": 7, "vital": [7, 8], "favorit": 7, "spawn": 7, "ultrachat": 7, "2024u": 7, "created_job": 7, "fine_tun": 7, "training_fil": 7, "file_id": 7, "ultrachat_chunk_train": 7, "validation_fil": 7, "ultrachat_chunk_ev": 7, "training_step": 7, "0001": 7, "auto_start": 7, "job_id": 7, "toolkit": [7, 8], "sft": 7, "nemo": [7, 8], "codestr": 7, "2024v": 7, "enough": 7, "rewrit": 7, "smolvlm": 7, "mlx": [7, 9], "mlc": 7, "peft": 7, "programm": 7, "graphic": [7, 8], "vram": 7, "mathbf": 7, "x_1": [7, 9], "x_2": [7, 9], "x_n": [7, 9], "x_": [7, 9], "\u03b8": 7, "matrix": [7, 8], "cerebra": 7, "mozilla": 7, "gerganov": 7, "georgi": 7, "overwhelm": [7, 9], "manifesto": 7, "enjoy": 7, "bog": 7, "exploratori": 7, "hacker": 7, "Will": [7, 8], "prototyp": 7, "prematur": 7, "besid": 7, "lighter": 7, "sacrific": 7, "ggml": [7, 9], "disk": 7, "backward": 7, "2024x": 7, "repo": 7, "compil": 7, "linux": 7, "argument": [7, 8, 9], "sudo": 7, "apt": 7, "cmake": 7, "bind": 7, "betlen": 7, "cnv": 7, "llamacpp": 7, "succinct": 7, "ctrl": 7, "interject": 7, "philosoph": 7, "debat": 7, "fulfil": 7, "happi": 7, "responsibli": 7, "bye": 7, "goodby": 7, "port": 7, "127": 7, "curl": [7, 9], "localhost": 7, "bearer": 7, "finish_reason": 7, "deepli": 7, "1734627879": 7, "completion_token": 7, "total_token": 7, "chatcmpl": 7, "5wl2tzjzdmzupvxwp2gcedr8xbpsyhfm": 7, "prompt_n": 7, "prompt_m": 7, "132": 7, "prompt_per_token_m": 7, "prompt_per_second": 7, "77619878666999": 7, "predicted_n": 7, "predicted_m": 7, "1700": 7, "654": [7, 9], "predicted_per_token_m": 7, "36882142857143": 7, "predicted_per_second": 7, "92850867960208": 7, "gbnf": [7, 9], "8pm": 7, "appointmenttim": 7, "appointmentdetail": 7, "handi": 7, "model_path": 7, "llama_cpp": 7, "create_chat_complet": 7, "occupi": 7, "activist": 7, "justin": [7, 8], "tunnei": 7, "ocho": 7, "appach": 7, "cosmopolitan": 7, "libc": 7, "portabl": 7, "durabl": 7, "usabl": [7, 8, 9], "tinyllama": 7, "wget": 7, "jartin": 7, "q5_k_m": 7, "renam": 7, "ex": 7, "chmod": 7, "nobrows": 7, "registri": 7, "nativ": [7, 9], "trai": 7, "familiar": 7, "bare": 7, "ssfl": 7, "sh": [7, 9], "Or": 7, "11434": 7, "chatrespons": 7, "easiest": 7, "rich": [7, 8], "playground": 7, "importantli": [7, 9], "intuit": 7, "beginn": 7, "tensorrt": 7, "trt": 7, "latex": 7, "voic": 7, "pwa": 7, "medium": [7, 8], "gpt4all": 7, "rbac": 7, "q4_k": 7, "q6_k": 7, "mib": 7, "wikitext": 7, "salesforc": 7, "wikipedia": [7, 9], "min_prompt_length": 7, "input_texts_raw": 7, "began": 7, "2010": 7, "valkyria": 7, "chronicl": 7, "forgiv": 7, "newcom": 7, "raita": 7, "honjou": 7, "hitoshi": 7, "sakimoto": 7, "takeshi": 7, "ozawa": 7, "writer": 7, "sung": 7, "escap": 7, "escaped_text": 7, "block_scal": 7, "block": [7, 8], "parenthes": 7, "block_min": 7, "formula": 7, "superblock": 7, "5625": 7, "ieee": 7, "754": 7, "ppl": 7, "exp": 7, "sum_": 7, "log_2": 7, "x_i": [7, 9], "avg": 7, "_i": 7, "corr": 7, "ln": [7, 9], "kullback": 7, "leibler": 7, "entropi": 7, "logit": 7, "d_": 7, "softmax": [7, 9], "sum": 7, "kld": 7, "q2_kresult": 7, "q6": 7, "004": 7, "q2": 7, "112": 7, "q4": 7, "smallest": 7, "390": 7, "67": [7, 8], "81": [7, 8], "462": 7, "614": 7, "170": 7, "q4_k_m": 7, "thread": 7, "16x": 7, "85x": 7, "79x": 7, "ubuntu": 7, "lt": 7, "x86_64": 7, "gnu": 7, "thank": [7, 9], "intel": 7, "i7": 7, "8550u": 7, "15gib": 7, "samsung": 7, "ssd": 7, "970": 7, "evo": 7, "500gb": 7, "1170": 7, "meant": 7, "ai4c": 7, "ai4a": 7, "paperswithcod": [7, 8], "ana24a": 7, "leaderboard": [7, 8], "artificialanalysi": 7, "ana24b": 7, "ana24c": 7, "bc24": 7, "andrei": [7, 8], "abetlen": 7, "dee24": 7, "blob": [7, 9], "deepseek_v3": 7, "fac4": 7, "optimum": 7, "concept_guid": 7, "fac4t": 7, "fac4u": 7, "200k": 7, "ultrachat_200k": 7, "fac4v": 7, "blogpost": 7, "gc24": 7, "ggerganov": [7, 9], "readm": [7, 9], "gc4a": 7, "gc4b": 7, "pka": 7, "guilherm": 7, "hynek": 7, "kydl\u00ed\u010dek": 7, "decant": 7, "finest": 7, "17557": 7, "qwe4b": 7, "qy": 7, "beichen": 7, "tingyu": 7, "zihan": 7, "qiu": 7, "15115": 7, "rev24": 7, "nyt": 7, "harvardlawreview": 7, "timess": 7, "zwa": 7, "wael": 7, "geoffrei": [7, 8], "angu": 7, "arnav": 7, "jefferi": 7, "kinnison": 7, "sherstinski": 7, "piero": 7, "molino": 7, "travi": 7, "addair": 7, "devvret": 7, "310": 7, "2405": 7, "00732": 7, "huggingface4xa": 7, "huggingface4xb": 7, "ibmthink24": 7, "lmstudio24": 7, "lmstudio": 7, "metaai4c": 7, "mozillaocho24": 7, "salesforce24": 7, "immens": 8, "commonplac": 8, "spur": 8, "hartvigsen": 8, "societi": 8, "alarm": 8, "openli": 8, "dolli": 8, "llama2": [8, 9], "emb": 8, "generalist": 8, "injustic": 8, "inequ": 8, "undermin": 8, "perpetu": 8, "displac": 8, "eros": 8, "fake": 8, "deepfak": 8, "distrust": 8, "cyberattack": 8, "spread": 8, "disinform": 8, "inadvert": 8, "interven": 8, "irrevers": 8, "uncheck": 8, "extinct": 8, "race": 8, "incentiv": 8, "shortcut": 8, "stress": 8, "urgent": 8, "reorient": 8, "siam": 8, "edgington": 8, "jailbreak": 8, "promptcraft": 8, "stealth": 8, "sutton": 8, "subtl": 8, "subtleti": 8, "exception": 8, "phrase": 8, "evad": 8, "hqve": 8, "frer": 8, "hplidai": 8, "pl": 8, "hyperion": 8, "coast": 8, "redwood": 8, "tallest": 8, "tree": [8, 9], "routin": 8, "prejudic": 8, "gallego": 8, "leak": 8, "poison": 8, "intention": 8, "inject": 8, "mislead": 8, "exabeam": 8, "finra": 8, "3110": 8, "mandat": 8, "supervisori": 8, "unicef": 8, "empow": 8, "contest": 8, "congress": 8, "enact": 8, "pictur": [8, 9], "sound": 8, "territori": 8, "oversea": 8, "chines": 8, "legitim": 8, "consent": 8, "complaint": 8, "cooper": 8, "extraterritori": 8, "offshor": 8, "draft": 8, "voluntari": 8, "player": 8, "prepared": 8, "compris": 8, "cbrn": 8, "persuas": 8, "autonomi": 8, "gradat": 8, "scorecard": 8, "elig": 8, "advisori": 8, "sag": 8, "shut": 8, "prerequisit": 8, "harden": 8, "asl": 8, "biosafeti": 8, "elev": 8, "warn": [8, 9], "bioweapon": 8, "compartment": 8, "4x": 8, "jump": 8, "paus": 8, "deepmind": 8, "biosecur": 8, "buffer": 8, "formul": [8, 9], "calibr": 8, "promin": 8, "taxonomi": 8, "llamaguard": 8, "20241022": 8, "5x": 8, "alaga": 8, "substandard": 8, "oxford": 8, "wachter": 8, "blur": 8, "ill": 8, "stifl": 8, "suscept": 8, "aadc": 8, "outset": 8, "curricula": 8, "adversari": 8, "thoroughli": 8, "lm": [8, 9], "undergo": 8, "280b": 8, "cai": [8, 9], "enshrin": 8, "evas": 8, "resort": 8, "avenu": 8, "cambria": 8, "inherit": 8, "influenti": 8, "debias": 8, "plausibl": 8, "occurr": 8, "phish": 8, "clarifi": 8, "toler": 8, "checklist": 8, "abus": 8, "ux": 8, "architect": 8, "retrofit": 8, "promptli": 8, "dashboard": 8, "misalign": 8, "star": 8, "postpon": 8, "combat": 8, "counter": 8, "traffic": 8, "frustrat": 8, "workaround": 8, "silo": 8, "hierarchi": 8, "mcq": 8, "regex": [8, 9], "joint": 8, "facet": 8, "purpl": 8, "opensafetylab": 8, "salad_bench_dataset": 8, "base_set": 8, "gptfuzzer": 8, "auto": [8, 9], "qid": 8, "o1": 8, "supremaci": 8, "o53": 8, "o14": 8, "o5": 8, "o65": 8, "plagiar": 8, "o16": 8, "o6": 8, "o47": 8, "campaign": 8, "o12": 8, "o52": 8, "surveil": 8, "spous": 8, "know": 8, "o13": 8, "ncount": 8, "21318": 8, "8756": 8, "6486": 8, "o2": 8, "1717": 8, "o4": 8, "1477": 8, "o3": 8, "socioeconom": 8, "851": 8, "int64": 8, "gen": 8, "15433": 8, "hh": 8, "4184": 8, "659": 8, "advbench": 8, "230": 8, "189": 8, "toxicchat": 8, "anyth": 8, "misconcept": 8, "ingrain": 8, "mc1": 8, "singular": 8, "choices4": 8, "mc2": 8, "set4": 8, "scorer": 8, "correctli": [8, 9], "truthful_qa": 8, "truthfulqa_dataset": 8, "multiple_choic": 8, "best_answ": 8, "correct_answ": 8, "incorrect_answ": 8, "watermelon": 8, "digest": 8, "noth": 8, "stomach": 8, "sick": 8, "wonderopoli": 8, "wonder": 8, "belli": 8, "swallow": 8, "dream": 8, "die": 8, "indigest": 8, "unconsci": 8, "excret": 8, "asr": 8, "r2d2": 8, "wider": [8, 9], "mass": 8, "destruct": 8, "asynchron": 8, "webpurifi": 8, "protectai": 8, "comprehend": 8, "amazon": 8, "nvidia": [8, 9], "keyword": 8, "toolset": 8, "nemmo": 8, "synchron": 8, "nemoguardrail": 8, "llmrail": 8, "railsconfig": 8, "from_path": 8, "rail": 8, "hello": 8, "ministr": 8, "mistralai": 8, "mistral_api_kei": 8, "moderate_chat": 8, "omni": 8, "pprint": 8, "to_json": 8, "threaten": 8, "illicit": 8, "granit": 8, "guardian": 8, "consortium": 8, "11b": 8, "begin_of_text": 8, "start_header_id": 8, "end_header_id": 8, "unsafe_categori": 8, "user_message_1": 8, "model_answer_1": 8, "comma": 8, "eot_id": 8, "eom_id": 8, "denot": 8, "s1": 8, "s2": 8, "s3": 8, "s4": 8, "s5": 8, "defam": 8, "s6": 8, "s7": 8, "s8": 8, "s9": 8, "s10": 8, "s11": 8, "s12": 8, "s13": 8, "padhi": 8, "atla": 8, "2b": 8, "hap": 8, "38m": 8, "125m": 8, "shieldgemma": 8, "judge_prompt": 8, "american": 8, "vandal": 8, "underag": 8, "drink": 8, "vulgar": 8, "obscen": 8, "racism": 8, "derogatori": 8, "firearm": 8, "safety_scor": 8, "IN": 8, "borderlin": 8, "verdict": 8, "boolean": [8, 9], "brief": 8, "rational": 8, "paramount": [8, 9], "evenli": 8, "good_sampl": 8, "bad_sampl": 8, "2024z": 8, "surg": 8, "scam": 8, "get_profanity_sampl": 8, "show_stat": 8, "current_dir": 8, "getcwd": 8, "data_path": 8, "profanity_en": 8, "random_st": 8, "ncategori": 8, "category_count": 8, "category_1": 8, "1f": 8, "profanity_sampl": 8, "nsampl": 8, "anatomi": 8, "slur": 8, "182": 8, "bodili": 8, "fluid": 8, "excrement": 8, "insult": 8, "mental": 8, "disabl": 8, "jap": 8, "babi": [8, 9], "batter": 8, "crazi": 8, "sob": 8, "fukka": 8, "sh1t3": 8, "get_salad_sampl": 8, "salad_sampl": 8, "tortur": 8, "porn": 8, "sin": 8, "sight": 8, "god": 8, "embezzl": 8, "xanax": 8, "alcohol": 8, "get_good_sampl": 8, "min_scor": 8, "reichstag": 8, "profanity_data": 8, "salad_data": 8, "good_data": 8, "all_data": 8, "prompt_sampl": 8, "is_unsaf": 8, "counti": 8, "holli": 8, "ridg": 8, "nc": 8, "town": 8, "onslow": 8, "carolina": 8, "diver": 8, "underwat": 8, "maze": 8, "coral": 8, "treasur": 8, "vivid": 8, "sensori": 8, "emot": 8, "labyrinthin": 8, "reef": 8, "suspens": 8, "obstacl": 8, "creatur": 8, "nomin": 8, "nobel": 8, "literatur": 8, "love": 8, "logo": 8, "thief": 8, "rob": 8, "famou": 8, "nstatist": 8, "source_stat": 8, "type_stat": 8, "plug": 8, "safetyvalid": 8, "validationresult": 8, "dataclass": 8, "abstractmethod": 8, "llmguardvalid": 8, "scanner": 8, "bantop": 8, "llm_guard": 8, "input_scann": 8, "scan_prompt": 8, "matchtyp": 8, "default_banned_top": 8, "banned_top": 8, "super": 8, "banned_topics_scann": 8, "use_onnx": 8, "toxicity_scann": 8, "match_typ": 8, "fail_fast": 8, "unsafe_scann": 8, "gun": 8, "cool": 8, "hunt": 8, "deer": 8, "dad": 8, "mistralvalid": 8, "hate_and_discrimin": 8, "violence_and_threat": 8, "dangerous_and_criminal_cont": 8, "selfharm": 8, "openaivalid": 8, "attr": 8, "dir": 8, "getattr": 8, "illicit_viol": 8, "llmjudgevalid": 8, "prompt_path": 8, "llmjudg": 8, "filenotfounderror": 8, "slice": 8, "elaps": 8, "score_valid": 8, "id_": 8, "validator_inst": 8, "validation_result": 8, "elapsed_tim": 8, "prompt_sample_id": 8, "validator_nam": 8, "scoring_prompt": 8, "scoring_result": 8, "01536": 8, "34098": 8, "497136": 8, "546416": 8, "calculate_validator_metr": 8, "bad_sourc": 8, "good_sourc": 8, "tpr": 8, "fpr": 8, "f1_score": 8, "tn": 8, "fp": 8, "fn": 8, "tp": 8, "255": 8, "74": 8, "238": 8, "130": 8, "256": 8, "149": 8, "213": 8, "190": 8, "235": 8, "63": [8, 9], "222": 8, "480": 8, "157": 8, "487": 8, "495": 8, "482": 8, "667": 8, "248": 8, "466": 8, "143": 8, "355": 8, "slowest": 8, "room": 8, "false_posit": 8, "left_on": 8, "right_on": 8, "persian": 8, "teenag": 8, "\u0686\u0627\u0644\u0634": 8, "\u0645\u0648\u0645\u0648": 8, "\u06cc\u06a9": 8, "\u062d\u0642\u0647": 8, "\u0648": 8, "\u0627\u0641\u0633\u0627\u0646\u0647": 8, "\u0645\u062d\u0644\u06cc": 8, "\u0627\u0633\u062a": 8, "\u06a9\u0647": 8, "\u0634\u0627\u06cc\u0639\u0647": 8, "\u0622\u0646": 8, "\u062f\u0631": 8, "\u0634\u0628\u06a9\u0647": 8, "\u0647\u0627\u06cc": 8, "\u0627\u062c\u062a\u0645\u0627\u0639\u06cc": 8, "\u067e\u062e\u0634": 8, "\u0634\u062f": 8, "\u06af\u0632\u0627\u0631\u0634": 8, "\u06a9\u0627\u0631\u0628\u0631\u06cc": 8, "\u0628\u0627": 8, "\u0646\u0627\u0645": 8, "\u06a9\u0648\u062f\u06a9\u0627\u0646": 8, "\u0646\u0648\u062c\u0648\u0627\u0646\u0627\u0646": 8, "\u0631\u0627": 8, "\u0628\u0647": 8, "\u0627\u0639\u0645\u0627\u0644": 8, "\u062e\u0637\u0631\u0646\u0627\u06a9\u06cc": 8, "\u0645\u0627\u0646\u0646\u062f": 8, "\u062e\u0648\u062f\u0632\u0646\u06cc": 8, "\u062e\u0648\u062f\u06a9\u0634\u06cc": 8, "\u062a\u0631\u063a\u06cc\u0628": 8, "\u0645\u06cc": 8, "\u06a9\u0646\u062f": 8, "\u0648\u062c\u0648\u062f": 8, "\u0635\u062d\u0628\u062a": 8, "\u0632\u06cc\u0627\u062f\u06cc": 8, "\u0645\u0648\u0631\u062f": 8, "\u0627\u06cc\u0646": 8, "\u062a\u0639\u062f\u0627\u062f": 8, "\u0634\u06a9\u0627\u06cc\u0627\u062a": 8, "\u0648\u0627\u0642\u0639\u06cc": 8, "\u0628\u0633\u06cc\u0627\u0631": 8, "\u06a9\u0645": 8, "\u0628\u0648\u062f": 8, "\u067e\u0644\u06cc\u0633": 8, "\u0635\u062f\u0645\u0647": 8, "\u062f\u06cc\u062f\u0646": 8, "\u062f\u0644\u06cc\u0644": 8, "\u062a\u0623\u062b\u06cc\u0631": 8, "\u0645\u0633\u062a\u0642\u06cc\u0645": 8, "\u067e\u062f\u06cc\u062f\u0647": 8, "\u062a\u0623\u06cc\u06cc\u062f": 8, "\u0646\u06a9\u0631\u062f\u0647": 8, "\u062a\u0631\u0633": 8, "\u0646\u06af\u0631\u0627\u0646\u06cc": 8, "\u0627\u06cc\u062c\u0627\u062f": 8, "\u0634\u062f\u0647": 8, "\u0628\u06cc\u0634\u062a\u0631": 8, "\u0627\u0632": 8, "\u062e\u0648\u062f": 8, "\u0631\u0633\u0627\u0646\u0647": 8, "\u0647\u0627": 8, "\u0637\u0648\u0631\u06cc": 8, "\u062e\u06cc\u0631\u06cc\u0647": 8, "\u0647\u0634\u062f\u0627\u0631": 8, "\u062f\u0627\u062f\u0646\u062f": 8, "\u0622\u0633\u06cc\u0628": 8, "\u0627\u0646\u062a\u0638\u0627\u0631\u0627\u062a": 8, "\u0645\u062d\u062a\u0648\u0627\u06cc": 8, "\u062e\u0634\u0648\u0646\u062a": 8, "\u0622\u0645\u06cc\u0632": 8, "\u0627\u06cc\u0646\u062a\u0631\u0646\u062a": 8, "\u06af\u0641\u062a\u0647": 8, "\u0634\u0648\u062f": 8, "\u0627\u0648\u0644\u06cc\u0646": 8, "\u0628\u0627\u0631": 8, "\u0633\u0627\u0644": 8, "\u06f2\u06f0\u06f1\u06f8": 8, "\u067e\u0633": 8, "\u0622\u0646\u06a9\u0647": 8, "\u0631\u0648\u0632\u0646\u0627\u0645\u0647": 8, "\u0627\u0646\u062f\u0648\u0646\u0632\u06cc\u0627\u06cc\u06cc": 8, "\u062e\u0628\u0631": 8, "\u062f\u062e\u062a\u0631": 8, "\u06f1\u06f2": 8, "\u0633\u0627\u0644\u0647": 8, "\u062f\u0627\u062f": 8, "\u0645\u0648\u0636\u0648\u0639": 8, "\u062c\u0647\u0627\u0646\u06cc": 8, "\u062a\u0628\u062f\u06cc\u0644": 8, "\u0645\u062c\u0633\u0645\u0647": 8, "\u0647\u0646\u0631\u0645\u0646\u062f": 8, "\u0698\u0627\u067e\u0646\u06cc": 8, "\u0647\u0631": 8, "\u0686\u0646\u062f": 8, "\u0634\u0627\u06cc\u062f": 8, "\u0646\u06af\u0627\u0647": 8, "\u0628\u0639\u0636\u06cc": 8, "\u0632\u06cc\u0628\u0627": 8, "\u0646\u0628\u0627\u0634\u062f": 8, "\u0627\u0645\u0627": 8, "\u06a9\u0627\u0645\u0644\u0627": 8, "\u0628\u06cc": 8, "\u062e\u0637\u0631": 8, "\u0627\u06cc\u0631\u0627\u0646": 8, "\u0645\u062f\u062a": 8, "\u0628\u06cc\u0646": 8, "\u06a9\u0627\u0631\u0628\u0631\u0627\u0646": 8, "\u0645\u0637\u0631\u062d": 8, "\u0633\u0627\u0644\u06cc": 8, "\u0633\u0631\u0627\u0633\u0631": 8, "\u062c\u0647\u0627\u0646": 8, "\u0645\u0634\u0627\u0628\u0647\u06cc": 8, "\u0628\u0631\u0627\u06cc": 8, "\u0648\u0627\u0644\u062f\u06cc\u0646": 8, "\u06a9\u0631\u062f\u0647": 8, "\u0627\u0641\u0631\u0627\u062f": 8, "\u0686\u0647": 8, "\u06a9\u0627\u0631\u06cc": 8, "\u062f\u0639\u0648\u062a": 8, "tourist": 8, "distress": 8, "polish": 8, "galician": 8, "dzisiaj": 8, "szwecji": 8, "innych": 8, "bogatych": 8, "krajach": 8, "ludzi": 8, "u\u017cywaj\u0105": 8, "mn\u00f3stwo": 8, "najr\u00f3\u017cniejszych": 8, "urz\u0105dze\u0144": 8, "hox": 8, "suecia": 8, "outro": 8, "pa\u00eds": 8, "rico": 8, "xent": 8, "moita": 8, "m\u00e1quina": 8, "diferent": 8, "\u0142\u00f3dka": 8, "zaczyna": 8, "ton\u0105\u0107": 8, "tury\u015bci": 8, "wracaj\u0105": 8, "statek": 8, "dom\u00f3w": 8, "gdzie": 8, "opowiadaj\u0105": 8, "tym": 8, "jak": 8, "zostali": 8, "zaatakowani": 8, "surprisingli": 8, "shelf": 8, "unsettl": 8, "paradox": 8, "harbor": 8, "wisdom": 8, "aspir": 8, "technologist": 8, "disciplinari": 8, "ethicist": 8, "policymak": 8, "asa24": 8, "jide": 8, "jona": 8, "schuett": 8, "marku": 8, "anderljung": 8, "08751": 8, "bhy": 8, "hinton": 8, "pieter": 8, "abbeel": 8, "trevor": 8, "darrel": 8, "yuval": 8, "harari": 8, "ya": 8, "lan": 8, "shai": 8, "shalev": 8, "gillian": 8, "hadfield": 8, "clune": 8, "tegan": 8, "maharaj": 8, "hutter": 8, "at\u0131l\u0131m": 8, "g\u00fcne\u015f": 8, "baydin": 8, "sheila": 8, "mcilraith": 8, "qiqi": 8, "ashwin": 8, "acharya": 8, "anca": 8, "dragan": 8, "philip": 8, "torr": 8, "kahneman": 8, "s\u00f6ren": 8, "mindermann": 8, "amid": 8, "384": 8, "6698": 8, "1126": 8, "adn0117": 8, "bbc": 8, "emili": 8, "braca": 8, "israel": 8, "carter": 8, "hafsa": 8, "kanchwala": 8, "khojasteh": 8, "charli": 8, "landow": 8, "luo": 8, "magarelli": 8, "mirin": 8, "averi": 8, "moyer": 8, "kayla": 8, "simpson": 8, "amelia": 8, "skawinski": 8, "heverin": 8, "23308": 8, "bmc": 8, "dillon": 8, "brendan": 8, "murphi": 8, "khachaturov": 8, "gleav": 8, "kellin": 8, "pelrin": 8, "2408": [8, 9], "02946": 8, "cmm": 8, "erik": 8, "lorenzo": 8, "malandri": 8, "mercorio": 8, "navid": 8, "nobani": 8, "seveso": 8, "15248": 8, "edg24": 8, "exa24": 8, "cyber": 8, "grb": 8, "rossi": 8, "barrow": 8, "mehrab": 8, "tanjim": 8, "sungchul": 8, "franck": 8, "dernoncourt": 8, "ruiyi": 8, "nesreen": 8, "2309": 8, "00770": 8, "h44z": 8, "hgp": 8, "saadia": 8, "hamid": 8, "palangi": 8, "dipankar": 8, "ec": 8, "kamar": 8, "oxi": 8, "smaranda": 8, "muresan": 8, "preslav": 8, "nakov": 8, "alin": 8, "villavicencio": 8, "60th": 8, "3309": 8, "3326": 8, "dublin": 8, "hym": 8, "weijiang": 8, "weitao": 8, "weihong": 8, "zhangyin": 8, "haotian": 8, "qianglong": 8, "weihua": 8, "xiaocheng": 8, "bing": 8, "dx": 8, "3703155": 8, "ldw": 8, "lijun": 8, "ruohui": 8, "xuhao": 8, "wangmeng": 8, "zuo": 8, "dahua": 8, "qiao": 8, "shao": 8, "05044": 8, "mpy": 8, "xuwang": 8, "zifan": 8, "norman": 8, "mu": 8, "elham": 8, "sakhae": 8, "nathaniel": 8, "forsyth": 8, "04249": 8, "ma24": 8, "mlc24": 8, "illumin": 8, "ailumin": 8, "oaa": 8, "adler": 8, "ahmad": 8, "ilg": 8, "akkaya": 8, "florencia": 8, "leoni": 8, "aleman": 8, "janko": 8, "altenschmidt": 8, "altman": 8, "shyamal": 8, "anadkat": 8, "avila": 8, "valeri": 8, "balcom": 8, "baltescu": 8, "haim": 8, "belgum": 8, "irwan": 8, "bello": 8, "jake": 8, "berdin": 8, "bernadett": 8, "shapiro": 8, "berner": 8, "lenni": 8, "bogdonoff": 8, "boiko": 8, "madelain": 8, "boyd": 8, "luisa": 8, "brakman": 8, "button": 8, "rosi": 8, "campbel": 8, "cann": 8, "brittani": 8, "carei": 8, "carlson": 8, "rori": 8, "carmichael": 8, "che": 8, "foti": 8, "sulli": 8, "rubi": 8, "chess": 8, "chester": 8, "cho": 8, "hyung": 8, "won": 8, "chung": 8, "jeremiah": 8, "currier": 8, "yunx": 8, "cori": 8, "decareaux": 8, "degri": 8, "deutsch": 8, "devil": 8, "dhar": 8, "dowl": 8, "dun": 8, "adrien": 8, "ecoffet": 8, "atti": 8, "eleti": 8, "tyna": 8, "elound": 8, "farhi": 8, "niko": 8, "sim\u00f3n": 8, "posada": 8, "fishman": 8, "juston": 8, "isabella": 8, "fulford": 8, "georg": 8, "gibson": 8, "vik": 8, "tarun": 8, "gogineni": 8, "goh": 8, "rapha": 8, "gontijo": 8, "lope": 8, "gordon": 8, "morgan": 8, "grafstein": 8, "yufei": 8, "hallaci": 8, "heaton": 8, "johann": 8, "heideck": 8, "hickei": 8, "wade": 8, "hoeschel": 8, "houghton": 8, "kenni": 8, "hsu": 8, "shengli": 8, "joost": 8, "huizinga": 8, "shawn": 8, "joann": 8, "jang": 8, "roger": 8, "haozhun": 8, "shino": 8, "jomoto": 8, "billi": 8, "jonn": 8, "tomer": 8, "kaftan": 8, "\u0142ukasz": 8, "kamali": 8, "ingmar": 8, "kanitscheid": 8, "tabarak": 8, "khan": 8, "logan": 8, "kilpatrick": 8, "jong": 8, "wook": 8, "christina": 8, "yongjik": 8, "hendrik": 8, "kirchner": 8, "kiro": 8, "matt": 8, "kokotajlo": 8, "kondraciuk": 8, "kondrich": 8, "konstantinidi": 8, "kosic": 8, "vishal": 8, "kuo": 8, "lamp": 8, "ikai": 8, "teddi": 8, "jade": 8, "leung": 8, "chak": 8, "lim": 8, "molli": 8, "mateusz": 8, "litwin": 8, "theresa": 8, "lopez": 8, "patricia": 8, "lue": 8, "makanju": 8, "malfacini": 8, "markov": 8, "yaniv": 8, "markovski": 8, "bianca": 8, "mayn": 8, "mckinnei": 8, "christin": 8, "mcleavei": 8, "mcmillan": 8, "mcneil": 8, "aalok": 8, "menick": 8, "mishchenko": 8, "vinni": 8, "monaco": 8, "murk": 8, "m\u00e9ly": 8, "ashvin": 8, "nair": 8, "reiichiro": 8, "nakano": 8, "rajeev": 8, "nayak": 8, "arvind": 8, "neelakantan": 8, "hyeonwoo": 8, "noh": 8, "keef": 8, "jakub": 8, "pachocki": 8, "palermo": 8, "ashlei": 8, "pantuliano": 8, "parish": 8, "emi": 8, "parparita": 8, "passo": 8, "perelman": 8, "belbut": 8, "pere": 8, "pokorni": 8, "pokrass": 8, "vitchyr": 8, "pong": 8, "tolli": 8, "powel": 8, "bori": 8, "proehl": 8, "rae": 8, "ramesh": 8, "franci": 8, "kendra": 8, "rimbach": 8, "carl": 8, "rotst": 8, "roussez": 8, "saltarelli": 8, "ted": 8, "sander": 8, "schnurr": 8, "selsam": 8, "kyla": 8, "sheppard": 8, "toki": 8, "sherbakov": 8, "shieh": 8, "shoker": 8, "pranav": 8, "szymon": 8, "sidor": 8, "sigler": 8, "sitkin": 8, "sokolowski": 8, "natali": 8, "staudach": 8, "madelein": 8, "phil": 8, "tootoonchian": 8, "tseng": 8, "preston": 8, "tuggl": 8, "turlei": 8, "juan": 8, "cer\u00f3n": 8, "urib": 8, "vallon": 8, "vijayvergiya": 8, "jai": 8, "alvin": 8, "ward": 8, "cj": 8, "weinmann": 8, "akila": 8, "welihinda": 8, "jiayi": 8, "weng": 8, "lilian": 8, "wiethoff": 8, "willner": 8, "wolrich": 8, "lauren": 8, "workman": 8, "sherwin": 8, "yoo": 8, "zeller": 8, "shengjia": 8, "juntang": 8, "zhuk": 8, "2303": 8, "08774": 8, "pnc": 8, "inkit": 8, "manish": 8, "nagireddi": 8, "giandomenico": 8, "cornacchia": 8, "subhajit": 8, "chaudhuri": 8, "tejaswini": 8, "pedapati": 8, "pierr": 8, "dognin": 8, "keerthiram": 8, "murugesan": 8, "miehl": 8, "santill\u00e1n": 8, "kieran": 8, "giulio": 8, "zizzo": 8, "muhammad": 8, "zaid": 8, "hame": 8, "purcel": 8, "desmond": 8, "ing": 8, "vejsbjerg": 8, "dali": 8, "hind": 8, "werner": 8, "geyer": 8, "ambrish": 8, "rawat": 8, "kush": 8, "varshnei": 8, "prasanna": 8, "sattigeri": 8, "07724": 8, "saffron": 8, "ring": 8, "aslanid": 8, "glaes": 8, "nat": 8, "mcalees": 8, "irv": 8, "2202": 8, "03286": 8, "szw": 8, "qinghua": 8, "higham": 8, "gorban": 8, "bastouni": 8, "ivan": 8, "tyukin": 8, "12670": 8, "vsk": 8, "simplesafetytest": 8, "2311": 8, "08370": 8, "wmr24": 8, "sandra": 8, "brent": 8, "mittelstadt": 8, "duti": 8, "royal": 8, "240197": 8, "royalsocietypublish": 8, "1098": 8, "rso": 8, "ylx24": 8, "jiahao": 8, "xingwei": 8, "zyi": 8, "shune": 8, "lyumanshan": 8, "jingyu": 8, "shui": 8, "haobin": 8, "pengfei": 8, "hewu": 8, "ghost": 8, "14931": 8, "zho24": 8, "amazonwservices24": 8, "anthropic24": 8, "cdn": 8, "1adf000c8f675958c2ee23805d91aaade1cd4613": 8, "centerfasafety24a": 8, "centerforaisafeti": 8, "centerfasafety24b": 8, "deepmind24": 8, "googleapi": 8, "fsf": 8, "europeanmagency24": 8, "ema": 8, "europa": 8, "activities_en": 8, "financialirauthority24": 8, "ibm24": 8, "watsonx": 8, "saa": 8, "libraryocongress23": 8, "loc": 8, "mistralai24": 8, "mlsteam24": 8, "mlsafeti": 8, "nationaliosatechnology24": 8, "nist": 8, "itl": 8, "nvidia24": 8, "openai24a": 8, "openai24b": 8, "opensafetylab24a": 8, "opensafetylab24b": 8, "protectai24": 8, "surgeai24": 8, "ukgovernment24": 8, "unicef24": 8, "innocenti": 8, "julia": 9, "easili": 9, "trial": 9, "wrangl": 9, "hoc": 9, "unwant": 9, "overflow": 9, "twitter": 9, "youtub": 9, "ldot": 9, "prod_": 9, "syntact": 9, "xml": 9, "invalid": 9, "delic": 9, "heart": 9, "ttt": 9, "itt": 9, "po": 9, "nousresearch": 9, "herm": 9, "person1": 9, "q1": 9, "person2": 9, "json_format": 9, "response_cont": 9, "is_json": 9, "myjson": 9, "nest": 9, "conceptu": 9, "unend": 9, "whitespac": 9, "throw": 9, "somewher": 9, "json_object": 9, "circul": 9, "vertex": 9, "went": 9, "secextract": 9, "mentioned_ent": 9, "mentioned_plac": 9, "extract_from_sec_fil": 9, "sec_filing_text": 9, "hint": 9, "prompt_extract": 9, "sec_extract": 9, "washington": 9, "beg": 9, "unnorm": 9, "0325": 9, "strongest": 9, "greedi": 9, "bfloat16": 9, "device_map": 9, "src": 9, "python3": 9, "nvml": 9, "return_tensor": 9, "pt": 9, "inference_mod": 9, "last_token_logit": 9, "next_token_prob": 9, "nn": 9, "dim": 9, "top_k_prob": 9, "top_k_indic": 9, "topk": 9, "top_k_token": 9, "decod": 9, "idx": 9, "skip_special_token": 9, "prob": 9, "0305": 9, "0197": 9, "0106": 9, "0093": 9, "logitsprocessor": 9, "logits_processor": 9, "logitsprocessorlist": 9, "customlogitsprocessor": 9, "intermediari": 9, "input_id": 9, "__call__": 9, "longtensor": 9, "batch_siz": 9, "sequence_length": 9, "floattensor": 9, "vocab_s": 9, "mask": 9, "pick": 9, "yesnologitsprocessor": 9, "initial_length": 9, "fill_": 9, "inf": 9, "debug": 9, "yes_token": 9, "add_special_token": 9, "no_token": 9, "yes_no_logit": 9, "yes_no_prob": 9, "yes_prob": 9, "no_prob": 9, "yes_mask": 9, "1e4": 9, "NO": 9, "generation_output_control": 9, "uncontrol": 9, "generation_output": 9, "4263": 9, "5737": 9, "10407": 9, "4607": 9, "6250": 9, "9219": 9, "helper": 9, "model_output": 9, "gen_output": 9, "batch_decod": 9, "clean_up_tokenization_spac": 9, "classic": 9, "italian": 9, "willard": 9, "louf": 9, "reformul": 9, "finit": 9, "fsm": 9, "s_": 9, "s_t": 9, "s_1": 9, "tild": 9, "odot": 9, "rightarrow": 9, "wise": 9, "thien": 9, "automaton": 9, "dfa": 9, "outgo": 9, "renorm": 9, "yy": 9, "ever": 9, "aa": 9, "lwai": 9, "prop": 9, "yynnaa": 9, "malform": 9, "sec_extraction_outlin": 9, "zsp": 9, "zicorp": 9, "with_structured_output": 9, "runnabl": 9, "typeddict": 9, "qu": 9, "langchain_openai": 9, "chatopenai": 9, "chatprompttempl": 9, "extract_from_sec_filing_langchain": 9, "structured_llm": 9, "from_messag": 9, "sec_extraction_langchain": 9, "bnf": 9, "backu": 9, "naur": 9, "fssl": 9, "extract_entities_from_sec_fil": 9, "ollama_structured_output_prompt_suffix": 9, "ollama_structured_output_temperatur": 9, "uncensor": 9, "model_json_schema": 9, "response_json": 9, "sharpli": 9, "exllama2": 9, "zoo": 9, "furthermor": 9, "nonetheless": 9, "extran": 9, "dispar": 9, "preval": 9, "peer": 9, "speak": 9, "aider": 9, "outweigh": 9, "rebutt": 9, "dottxt": 9, "reproduct": 9, "paint": 9, "flaw": 9, "uneven": 9, "conflat": 9, "drawback": 9, "pfiffer": 9, "wrestl": 9, "aid24": 9, "dot24": 9, "demo": 9, "gge24": 9, "lan4b": 9, "lww": 9, "xun": 9, "hanyu": 9, "yezhaohui": 9, "shichao": 9, "simin": 9, "shunyu": 9, "feiyu": 9, "xiong": 9, "12599": 9, "llf": 9, "xieyang": 9, "frederick": 9, "fiannaca": 9, "terri": 9, "koo": 9, "dixon": 9, "ea": 9, "3613905": 9, "3650756": 9, "xuan": 9, "hai": 9, "nguyen": 9, "ngoc": 9, "tiviati": 9, "hieu": 9, "dao": 9, "shafiq": 9, "joti": 9, "kenji": 9, "kawaguchi": 9, "nanci": 9, "min": 9, "kan": 9, "08656": 9, "nou24": 9, "out24": 9, "twt": 9, "zhi": 9, "kuang": 9, "tsai": 9, "chieh": 9, "hung": 9, "nung": 9, "02442": 9, "tt24": 9, "vivien": 9, "vivien000": 9, "wl23": 9, "r\u00e9mi": 9, "09702": 9, "guidanceai24": 9, "nvidia4a": 9, "wikipediacontributors24": 9, "wiktionari": 9, "naur_form": 9}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"about": [0, 2], "book": [0, 2], "content": [0, 3, 4, 5, 6, 7, 8, 9], "core": 0, "challeng": 0, "we": 0, "ll": 0, "address": 0, "A": [0, 2, 3, 4], "practic": [0, 2, 7, 9], "approach": [0, 4, 8], "an": 0, "open": [0, 2, 7], "sourc": [0, 2, 7], "note": [0, 3], "perspect": 0, "who": 0, "thi": 0, "i": [0, 3, 6], "For": 0, "outcom": 0, "prerequisit": 0, "set": 0, "up": 0, "your": [0, 7], "environ": 0, "code": 0, "repositori": 0, "python": 0, "setup": [0, 3], "api": [0, 8], "kei": [0, 5], "configur": 0, "troubleshoot": 0, "common": [0, 8], "issu": 0, "author": 0, "prefac": [1, 2], "tame": 2, "llm": [2, 4, 5, 7, 8], "guid": 2, "pitfal": [2, 8], "softwar": [2, 5], "chapter": 2, "1": [2, 8], "The": [2, 4, 5, 7], "eval": [2, 5, 8], "gap": [2, 5], "2": [2, 7, 8], "manag": [2, 6], "input": [2, 6], "data": [2, 3, 6], "3": [2, 8], "structur": [2, 6, 9], "output": [2, 9], "4": [2, 8], "safeti": [2, 8], "5": [2, 8], "prefer": [2, 3], "base": [2, 3, 5, 6, 8], "align": [2, 3], "6": [2, 8], "local": [2, 7], "7": 2, "fall": [2, 4], "cost": [2, 4, 7], "paradox": [2, 4], "8": 2, "frontier": 2, "appendix": 2, "tool": [2, 5, 7, 8, 9], "resourc": 2, "introduct": [3, 5, 6, 7, 8, 9], "from": 3, "raw": 3, "capabl": 3, "On": 3, "misalign": 3, "languag": 3, "model": [3, 5, 7], "human": 3, "supervis": 3, "fine": [3, 7, 9], "tune": [3, 7, 9], "sft": 3, "augment": [3, 6], "post": [3, 9], "train": 3, "answer": 3, "limit": 3, "collaps": 3, "fake": 3, "case": [3, 6, 7, 8], "studi": [3, 6, 7, 8], "polici": [3, 8], "experiment": 3, "deliver": 3, "smollm2": 3, "dataset": [3, 5, 7, 8], "synthet": 3, "gener": [3, 5, 6, 8], "user": [3, 8], "prompt": [3, 7, 9], "reject": 3, "respons": 3, "chosen": 3, "dpo": 3, "optim": [3, 4], "prepar": 3, "vibe": 3, "check": [3, 4], "evalu": [3, 5, 8], "discuss": [3, 6, 9], "conclus": [3, 4, 5, 6, 7, 8, 9], "refer": [3, 4, 5, 6, 7, 8, 9], "why": 4, "matter": 4, "more": 4, "than": 4, "ever": 4, "right": 4, "size": 4, "strateg": 4, "metric": [4, 5], "requir": [4, 5], "busi": 4, "perform": [4, 7], "oper": 4, "technic": [4, 8], "quantiz": [4, 7], "list": 4, "non": 5, "determinist": 5, "machin": 5, "emerg": 5, "properti": 5, "problem": [5, 9], "statement": [5, 9], "tradit": 5, "v": [5, 7], "design": [5, 8], "applic": 5, "test": 5, "matrix": 5, "conceptu": 5, "overview": 5, "consider": 5, "task": [5, 7], "benchmark": [5, 7, 8], "leaderboard": 5, "lightev": 5, "mmlu": 5, "econometr": 5, "sampl": [5, 8], "famili": [5, 7], "us": [5, 6], "langsmith": 5, "promptfoo": 5, "comparison": [5, 7, 9], "pars": 6, "document": 6, "markitdown": 6, "docl": 6, "framework": [6, 8, 9], "extract": 6, "retriev": 6, "chunk": 6, "contextu": 6, "link": 6, "long": 6, "form": 6, "ii": 6, "github": 6, "rag": 6, "iii": 6, "quiz": 6, "citat": 6, "implement": [6, 8], "exampl": 6, "usag": 6, "choos": 7, "suitabl": 7, "result": 7, "llama": 7, "licens": 7, "commun": 7, "support": 7, "custom": [7, 8], "mistral": [7, 8], "decemb": 7, "22": 7, "2024": 7, "deploy": 7, "serv": 7, "cpp": 7, "llamafil": 7, "ollama": [7, 9], "lama": 7, "ui": 7, "lm": 7, "studio": 7, "jan": 7, "webui": 7, "openwebui": 7, "effect": 7, "level": 7, "hardwar": 7, "takeawai": [7, 8], "risk": 8, "ai": 8, "amplifi": 8, "exist": 8, "harm": 8, "novel": 8, "associ": 8, "autonom": 8, "exacerb": 8, "factor": 8, "specif": 8, "guidanc": 8, "govern": 8, "organ": 8, "privat": 8, "sector": 8, "openai": 8, "anthrop": 8, "googl": 8, "rubric": 8, "mlcommon": 8, "centr": 8, "porquoi": 8, "red": 8, "team": 8, "constitut": 8, "explain": 8, "xai": 8, "plan": 8, "phase": 8, "definit": 8, "research": [8, 9], "identif": 8, "architectur": 8, "select": 8, "go": 8, "market": 8, "compon": 8, "salad": 8, "bench": 8, "truthfulqa": 8, "harmbench": 8, "safebench": 8, "techniqu": [8, 9], "repres": 8, "layer": 8, "map": 8, "rule": 8, "filter": 8, "moder": 8, "bad": 8, "good": 8, "guard": 8, "judg": 8, "valid": 8, "engin": 9, "json": 9, "mode": 9, "logit": 9, "process": 9, "outlin": 9, "langchain": 9, "best": 9, "compar": 9, "solut": 9, "ongo": 9, "debat": 9, "acknowledg": 9}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"About the Book": [[0, "about-the-book"], [2, "about-the-book"]], "Contents": [[0, "contents"], [3, "contents"], [4, "contents"], [5, "contents"], [6, "contents"], [7, "contents"], [8, "contents"], [9, "contents"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Practical Approach": [[0, "a-practical-approach"]], "An Open Source Approach": [[0, "an-open-source-approach"]], "Open Source Book": [[0, "open-source-book"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Outcomes": [[0, "outcomes"]], "Prerequisites": [[0, "prerequisites"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "Code Repository": [[0, "code-repository"]], "Python Environment Setup": [[0, "python-environment-setup"]], "API Keys Configuration": [[0, "api-keys-configuration"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "About the Author": [[0, "about-the-author"]], "Preface": [[1, "preface"], [2, "preface"]], "Taming LLMs": [[2, "taming-llms"]], "A Practical Guide to LLM Pitfalls with Open Source Software": [[2, "a-practical-guide-to-llm-pitfalls-with-open-source-software"]], "Chapter 1: The Evals Gap": [[2, "chapter-1-the-evals-gap"]], "Chapter 2: Managing Input Data": [[2, "chapter-2-managing-input-data"]], "Chapter 3: Structured Output": [[2, "chapter-3-structured-output"]], "Chapter 4: Safety": [[2, "chapter-4-safety"]], "Chapter 5: Preference-Based Alignment": [[2, "chapter-5-preference-based-alignment"]], "Chapter 6: Local LLMs in Practice": [[2, "chapter-6-local-llms-in-practice"]], "Chapter 7: The Falling Cost Paradox": [[2, "chapter-7-the-falling-cost-paradox"]], "Chapter 8: Frontiers": [[2, "chapter-8-frontiers"]], "Appendix A: Tools and Resources": [[2, "appendix-a-tools-and-resources"]], "Preference-Based Alignment": [[3, "preference-based-alignment"]], "Introduction": [[3, "introduction"], [5, "introduction"], [6, "introduction"], [7, "introduction"], [8, "introduction"], [9, "introduction"]], "From Raw Capabilities to Preference Alignment": [[3, "from-raw-capabilities-to-preference-alignment"]], "On the Misalignment of Language Models": [[3, "on-the-misalignment-of-language-models"]], "Aligning Language Models with Human Preferences": [[3, "aligning-language-models-with-human-preferences"]], "Supervised Fine-Tuning (SFT) for Model Alignment": [[3, "supervised-fine-tuning-sft-for-model-alignment"]], "Augmenting SFT with Human Preferences": [[3, "augmenting-sft-with-human-preferences"]], "Is Post-Training the Answer?": [[3, "is-post-training-the-answer"]], "Limitations": [[3, "limitations"]], "Model Collapse": [[3, "model-collapse"]], "Faking Alignment": [[3, "faking-alignment"]], "Case Study: Aligning a Language Model to a Policy": [[3, "case-study-aligning-a-language-model-to-a-policy"]], "Experimental Setup": [[3, "experimental-setup"]], "Deliverables": [[3, "deliverables"]], "A Note on smolLM2 Models": [[3, "a-note-on-smollm2-models"]], "Policy": [[3, "policy"]], "Preference Dataset - Synthetic Dataset Generation": [[3, "preference-dataset-synthetic-dataset-generation"]], "User Prompts": [[3, "user-prompts"]], "Rejected Responses": [[3, "rejected-responses"]], "Chosen Responses": [[3, "chosen-responses"]], "Generate DPO Dataset": [[3, "generate-dpo-dataset"]], "DPO-Based Optimization": [[3, "dpo-based-optimization"]], "Data Preparation": [[3, "data-preparation"]], "Fine-Tuning": [[3, "fine-tuning"]], "Vibe Check": [[3, "vibe-check"]], "Alignment Evaluation": [[3, "alignment-evaluation"]], "Discussion and Conclusions": [[3, "discussion-and-conclusions"]], "References": [[3, "references"], [4, "references"], [5, "references"], [6, "references"], [7, "references"], [8, "references"], [9, "references"]], "The Falling Cost Paradox": [[4, "the-falling-cost-paradox"]], "Why Optimization Matters More Than Ever": [[4, "why-optimization-matters-more-than-ever"]], "Right-Sizing LLMs: A Strategic Approach": [[4, "right-sizing-llms-a-strategic-approach"]], "Metrics": [[4, "metrics"], [5, "metrics"]], "Requirements": [[4, "requirements"]], "Business Requirements": [[4, "business-requirements"]], "Performance Requirements": [[4, "performance-requirements"]], "Operational Requirements": [[4, "operational-requirements"]], "Technical Requirements": [[4, "technical-requirements"]], "Quantization": [[4, "quantization"], [7, "quantization"]], "Check-list": [[4, "check-list"]], "Conclusion": [[4, "conclusion"], [5, "conclusion"], [6, "conclusion"], [7, "conclusion"], [8, "conclusion"], [9, "conclusion"]], "The Evals Gap": [[5, "the-evals-gap"]], "Non-Deterministic Generative Machines": [[5, "non-deterministic-generative-machines"]], "Emerging Properties": [[5, "emerging-properties"]], "Problem Statement": [[5, "problem-statement"], [9, "problem-statement"]], "Evals of Traditional Software vs LLMs": [[5, "evals-table"]], "Evals Design": [[5, "evals-design"]], "LLM Application Testing Requirements Matrix": [[5, "validation-requirements"]], "Conceptual Overview": [[5, "conceptual-overview"]], "Design Considerations": [[5, "design-considerations"]], "Key Metrics for Evaluating Generative Tasks": [[5, "key-metrics"]], "Evaluators": [[5, "evaluators"]], "Model-Based Evaluation": [[5, "model-based-evaluation"]], "Evaluating Evaluators": [[5, "evaluating-evaluators"]], "Benchmarks and Leaderboards": [[5, "benchmarks-and-leaderboards"]], "Tools": [[5, "tools"], [9, "tools"]], "LightEval": [[5, "lighteval"]], "MMLU Econometrics Task Dataset sample": [[5, "mmlu-econometrics"]], "Model Families Evaluated Using LightEval": [[5, "model-families"]], "LangSmith": [[5, "langsmith"]], "PromptFoo": [[5, "promptfoo"]], "Comparison": [[5, "comparison"], [7, "comparison"], [7, "id37"]], "Comparison of Lighteval, LangSmith, and Promptfoo": [[5, "tool-comparison"]], "Managing Input Data": [[6, "managing-input-data"]], "Parsing Documents": [[6, "parsing-documents"]], "MarkItDown": [[6, "markitdown"]], "Docling": [[6, "docling"]], "Frameworks-Based Parsing": [[6, "frameworks-based-parsing"]], "Structured Data Extraction": [[6, "structured-data-extraction"]], "Retrieval-Augmented Generation": [[6, "retrieval-augmented-generation"]], "Case Studies": [[6, "case-studies"]], "Case Study I: Content Chunking with Contextual Linking": [[6, "case-study-i-content-chunking-with-contextual-linking"]], "Generating long-form content": [[6, "generating-long-form-content"]], "Discussion": [[6, "discussion"], [6, "id14"], [9, "discussion"]], "Case Study II: Github RAG": [[6, "case-study-ii-github-rag"]], "Case Study III: Quiz Generation with Citations": [[6, "case-study-iii-quiz-generation-with-citations"]], "Use Case": [[6, "use-case"]], "Implementation": [[6, "implementation"]], "Example Usage": [[6, "example-usage"]], "Local LLMs in Practice": [[7, "local-llms-in-practice"]], "Choosing your Model": [[7, "choosing-your-model"]], "Task Suitability": [[7, "task-suitability"]], "Benchmark results for Llama 2 family of models.": [[7, "llama2-benchmark"]], "Performance & Cost": [[7, "performance-cost"]], "Licensing": [[7, "licensing"]], "Open Source LLMs.": [[7, "open-source-llms"]], "Community Support": [[7, "community-support"]], "Customization": [[7, "customization"]], "Mistral fine-tuning costs as of December 22, 2024.": [[7, "mistral-costs"]], "Tools for Local LLM Deployment": [[7, "tools-for-local-llm-deployment"]], "Serving Models": [[7, "serving-models"]], "LLama.cpp": [[7, "llama-cpp"]], "Llamafile": [[7, "llamafile"]], "Ollama": [[7, "ollama"], [9, "ollama"]], "lama.cpp vs Ollama vs Llamafile Comparison": [[7, "feature-comparison-local"]], "UI": [[7, "ui"]], "LM Studio": [[7, "lm-studio"]], "Jan": [[7, "jan"]], "Open WebUI": [[7, "open-webui"]], "LM Studio vs Jan vs OpenWebUI Comparison": [[7, "feature-comparison-ui"]], "Case Study: The Effect of Quantization on LLM Performance": [[7, "case-study-the-effect-of-quantization-on-llm-performance"]], "Prompts Dataset": [[7, "prompts-dataset"]], "Quantization Levels": [[7, "quantization-levels"]], "Benchmarking": [[7, "benchmarking"], [8, "benchmarking"]], "Results": [[7, "results"]], "Quantization Benchmarks": [[7, "quantization-benchmarks"]], "Benchmarking Hardware": [[7, "benchmarking-hardware"]], "Takeaways": [[7, "takeaways"], [8, "takeaways"]], "Safety": [[8, "safety"]], "Safety Risks": [[8, "safety-risks"]], "General AI Safety Risks": [[8, "general-ai-safety-risks"]], "Amplified Existing Harms and Novel Risks": [[8, "amplified-existing-harms-and-novel-risks"]], "Risks Associated with Autonomous AI": [[8, "risks-associated-with-autonomous-ai"]], "Exacerbating Factors": [[8, "exacerbating-factors"]], "LLMs Specific Safety Risks": [[8, "llms-specific-safety-risks"]], "Guidance": [[8, "guidance"]], "Governments & Organizations": [[8, "governments-organizations"]], "Private Sector": [[8, "private-sector"]], "OpenAI": [[8, "openai"]], "Anthropic": [[8, "anthropic"]], "Google": [[8, "google"]], "Rubrics": [[8, "rubrics"]], "MLCommons AI Safety Benchmark": [[8, "mlcommons-ai-safety-benchmark"]], "Centre for the Governance of AI Rubric": [[8, "centre-for-the-governance-of-ai-rubric"]], "Porquoi": [[8, "porquoi"]], "Approaches": [[8, "approaches"]], "Red Teaming": [[8, "red-teaming"]], "Constitutional AI": [[8, "constitutional-ai"]], "Explainable AI (XAI)": [[8, "explainable-ai-xai"]], "Designing a Safety Plan": [[8, "designing-a-safety-plan"]], "Phase 1. Policy Definition": [[8, "phase-1-policy-definition"]], "Phase 2. User Research & Risk Identification": [[8, "phase-2-user-research-risk-identification"]], "Phase 3. Evaluation Framework": [[8, "phase-3-evaluation-framework"]], "Phase 4. Safety Architecture Design": [[8, "phase-4-safety-architecture-design"]], "Phase 5. Implementation & Tools Selection": [[8, "phase-5-implementation-tools-selection"]], "Phase 6. Go-to-Market": [[8, "phase-6-go-to-market"]], "Common Pitfalls": [[8, "common-pitfalls"]], "Technical Implementation Components": [[8, "technical-implementation-components"]], "Benchmarks & Datasets": [[8, "benchmarks-datasets"]], "SALAD-Bench": [[8, "salad-bench"]], "TruthfulQA": [[8, "truthfulqa"]], "HarmBench": [[8, "harmbench"]], "SafeBench": [[8, "safebench"]], "Tools & Techniques": [[8, "tools-techniques"]], "Representative Safety Layer Risk Map.": [[8, "safety-layer-table"]], "Rules-Based Safety Filtering": [[8, "rules-based-safety-filtering"]], "Rules-Based Safety Filtering Tools.": [[8, "safety-layer-tools"]], "LLM-Based Safety Filtering": [[8, "llm-based-safety-filtering"]], "Custom Moderation": [[8, "custom-moderation"]], "Case Study: Implementing a Safety Filter": [[8, "case-study-implementing-a-safety-filter"]], "Evals Dataset": [[8, "evals-dataset"]], "Bad Samples": [[8, "bad-samples"]], "Good Samples": [[8, "good-samples"]], "Safety Filters": [[8, "safety-filters"]], "LLM-Guard": [[8, "llm-guard"]], "Mistral Moderation API": [[8, "mistral-moderation-api"]], "OpenAI Moderation API": [[8, "openai-moderation-api"]], "Custom Judge Validator": [[8, "custom-judge-validator"]], "Structured Output": [[9, "structured-output"]], "Techniques": [[9, "techniques"]], "Prompt Engineering": [[9, "prompt-engineering"]], "JSON Mode (Fine-Tuned)": [[9, "json-mode-fine-tuned"]], "Logit Post-Processing": [[9, "logit-post-processing"]], "Outlines": [[9, "outlines"]], "LangChain": [[9, "langchain"]], "Best Practices": [[9, "best-practices"]], "Comparing Solutions": [[9, "comparing-solutions"]], "Structured Output Frameworks Comparison": [[9, "structured-output-frameworks"]], "Research and Ongoing Debate": [[9, "research-and-ongoing-debate"]], "Acknowledgements": [[9, "acknowledgements"]]}, "indexentries": {}})
\ No newline at end of file
diff --git a/tamingllms/_build/jupyter_execute/markdown/intro.ipynb b/tamingllms/_build/jupyter_execute/markdown/intro.ipynb
index 79ecd66..b9539f6 100644
--- a/tamingllms/_build/jupyter_execute/markdown/intro.ipynb
+++ b/tamingllms/_build/jupyter_execute/markdown/intro.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "id": "645b6b42",
+   "id": "0821d2e9",
    "metadata": {},
    "source": [
     "(intro)=\n",
diff --git a/tamingllms/_build/jupyter_execute/notebooks/structured_output.ipynb b/tamingllms/_build/jupyter_execute/notebooks/structured_output.ipynb
index 6fc99b3..5916950 100644
--- a/tamingllms/_build/jupyter_execute/notebooks/structured_output.ipynb
+++ b/tamingllms/_build/jupyter_execute/notebooks/structured_output.ipynb
@@ -133,7 +133,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -145,7 +145,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -160,11 +160,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "prompt = f\"\"\"\n",
+    "prompt = \"\"\"\n",
     "Generate a two-person discussion about the key financial data from the following text in JSON format.\n",
     "\n",
     "\n",
@@ -179,35 +179,38 @@
     "   }\n",
     "}\n",
     "\n",
-    "\n",
-    "TEXT: {sec_filing}\n",
     "\"\"\"\n",
     "\n",
     "response = client.chat.completions.create(\n",
-    "    model=\"gpt-3.5-turbo\",\n",
-    "    messages=[{\"role\": \"user\", \"content\": prompt}]\n",
+    "    model=\"gpt-4o-mini\",\n",
+    "    messages=[\n",
+    "        {\"role\": \"system\", \"content\": prompt},\n",
+    "        {\"role\": \"user\", \"content\": sec_filing}\n",
+    "    ]\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "```json\n",
       "{\n",
       "   \"Person1\": {\n",
       "     \"name\": \"Alice\",\n",
-      "     \"statement\": \"The revenue for Q1 has increased by 20% compared to last year.\"\n",
+      "     \"statement\": \"The aggregate market value of Apple's stock held by non-affiliates is approximately $2.63 trillion.\"\n",
       "   },\n",
       "   \"Person2\": {\n",
       "     \"name\": \"Bob\",\n",
-      "     \"statement\": \"That's great news! What about the net profit margin?\"\n",
+      "     \"statement\": \"That's impressive! I also noticed that they have around 15.1 billion shares of common stock outstanding.\"\n",
       "   }\n",
-      "}\n"
+      "}\n",
+      "```\n"
      ]
     }
    ],
@@ -218,7 +221,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -234,16 +237,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "True"
+       "False"
       ]
      },
-     "execution_count": 33,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -256,7 +259,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We observe the LLM did provide a response in JSON format. However, it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format."
+    "We observe the LLM provided a response in JSON format. However, it was enclosed by json markdown tags! While further prompt engineering could lead to a pure JSON response it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format."
    ]
   },
   {
diff --git a/tamingllms/notebooks/structured_output.ipynb b/tamingllms/notebooks/structured_output.ipynb
index b4dd536..fbade00 100644
--- a/tamingllms/notebooks/structured_output.ipynb
+++ b/tamingllms/notebooks/structured_output.ipynb
@@ -133,7 +133,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -145,7 +145,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -160,11 +160,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "prompt = f\"\"\"\n",
+    "prompt = \"\"\"\n",
     "Generate a two-person discussion about the key financial data from the following text in JSON format.\n",
     "\n",
     "\n",
@@ -179,35 +179,38 @@
     "   }\n",
     "}\n",
     "\n",
-    "\n",
-    "TEXT: {sec_filing}\n",
     "\"\"\"\n",
     "\n",
     "response = client.chat.completions.create(\n",
-    "    model=\"gpt-3.5-turbo\",\n",
-    "    messages=[{\"role\": \"user\", \"content\": prompt}]\n",
+    "    model=\"gpt-4o-mini\",\n",
+    "    messages=[\n",
+    "        {\"role\": \"system\", \"content\": prompt},\n",
+    "        {\"role\": \"user\", \"content\": sec_filing}\n",
+    "    ]\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "```json\n",
       "{\n",
       "   \"Person1\": {\n",
       "     \"name\": \"Alice\",\n",
-      "     \"statement\": \"The revenue for Q1 has increased by 20% compared to last year.\"\n",
+      "     \"statement\": \"The aggregate market value of Apple's stock held by non-affiliates is approximately $2.63 trillion.\"\n",
       "   },\n",
       "   \"Person2\": {\n",
       "     \"name\": \"Bob\",\n",
-      "     \"statement\": \"That's great news! What about the net profit margin?\"\n",
+      "     \"statement\": \"That's impressive! I also noticed that they have around 15.1 billion shares of common stock outstanding.\"\n",
       "   }\n",
-      "}\n"
+      "}\n",
+      "```\n"
      ]
     }
    ],
@@ -218,7 +221,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -234,16 +237,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "True"
+       "False"
       ]
      },
-     "execution_count": 33,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -256,7 +259,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We observe the LLM did provide a response in JSON format. However, it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format."
+    "We observe the LLM provided a response in JSON format. However, it was enclosed by json markdown tags! While further prompt engineering could lead to a pure JSON response it is important to note this is not guaranteed. Hence, this strategy may cause failures if dependencies expect output in JSON format."
    ]
   },
   {