From 65923a8ede3ad5467264039fac8040976c1ca139 Mon Sep 17 00:00:00 2001
From: Georgi Gerganov <ggerganov@gmail.com>
Date: Sun, 10 Dec 2023 14:17:46 +0200
Subject: [PATCH] convert : determine n_ctx correctly

---
 convert.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/convert.py b/convert.py
index d36f25946..532c8b5ab 100755
--- a/convert.py
+++ b/convert.py
@@ -261,15 +261,15 @@ class Params:
         n_experts_used = None
 
         # hack to determine LLaMA v1 vs v2 vs CodeLlama
-        if config.get("rope_theta") == 1000000:
+        if config.get("moe"):
+            # Mixtral
+            n_ctx = 32768
+        elif config.get("rope_theta") == 1000000:
             # CodeLlama
             n_ctx = 16384
         elif config["norm_eps"] == 1e-05:
             # LLaMA v2
             n_ctx = 4096
-        elif config["moe"]:
-            # Mixtral
-            n_ctx = 32768
         else:
             # LLaMA v1
             n_ctx = 2048