Update app.py

app.py

@@ -9,7 +9,9 @@ FILENAME = "qwen2.5-3b-instruct-q4_k_m.gguf"
 MODEL_DIR = "model"
 MODEL_PATH = os.path.join(MODEL_DIR, FILENAME)
 
-
+# -----------------------
+# MODEL DOWNLOAD
+# -----------------------
 def download_model():
     print("🔽 Model indiriliyor...")
     os.makedirs(MODEL_DIR, exist_ok=True)
@@ -27,7 +29,6 @@ def download_model():
     print("✅ Model indirildi:", dst)
 
 
-# MODEL VAR MI?
 if not os.path.exists(MODEL_PATH):
     download_model()
 else:
@@ -35,41 +36,66 @@ else:
 
 print("⚙️ Model yükleniyor...")
 
+# -----------------------
+# LOAD MODEL
+# -----------------------
 llm = Llama(
     model_path=MODEL_PATH,
     n_ctx=4096,
     n_threads=4,
+    temperature=0.7,
+    repeat_penalty=1.1,
+    max_tokens=512,
     verbose=False
 )
 
 print("🚀 Model başarıyla yüklendi!")
 
 
-# CHAT FONKSİYONU
-def chat(message, history):
-    formatted_history = ""
+# -----------------------
+# STREAMING CHAT FUNCTION
+# -----------------------
+def stream_chat(message, history):
+    # Qwen Chat Formatı
+    system_prompt = "You are a helpful AI assistant."
+
+    messages = [{"role": "system", "content": system_prompt}]
+    
     for user, bot in history:
-        formatted_history += f"User: {user}\nAssistant: {bot}\n"
+        messages.append({"role": "user", "content": user})
+        messages.append({"role": "assistant", "content": bot})
 
-    prompt = formatted_history + f"User: {message}\nAssistant:"
+    messages.append({"role": "user", "content": message})
 
-    response = llm(prompt)
-    answer = response["choices"][0]["text"]
+    # Token Token Out Stream
+    output_stream = llm.create_chat_completion(
+        messages=messages,
+        stream=True
+    )
 
-    return answer
+    partial = ""
+    for chunk in output_stream:
+        delta = chunk["choices"][0]["delta"]
+        if "content" in delta:
+            partial += delta["content"]
+            yield partial  # canlı yazdır
 
 
-# GRADIO ARAYÜZ
+# -----------------------
+# GRADIO UI
+# -----------------------
 with gr.Blocks() as app:
-    gr.Markdown("# 🤖 DeeDe LLM (Qwen2.5-3B)")
+    gr.Markdown("# 🤖 DeeDe LLM (Qwen2.5-3B) — **STREAMING ENABLED** 🚀")
 
     chatbot = gr.Chatbot()
     msg = gr.Textbox(label="Mesajını Yaz")
 
     def respond(message, chat_history):
-        bot_response = chat(message, chat_history)
-        chat_history.append((message, bot_response))
-        return "", chat_history
+        stream = stream_chat(message, chat_history)
+        bot_message = ""
+        for token in stream:
+            bot_message = token
+            yield "", chat_history + [(message, bot_message)]
 
     msg.submit(respond, [msg, chatbot], [msg, chatbot])