Update app.py

app.py

@@ -4,7 +4,6 @@ from bert_score import score as bert_score
 import jieba
 import traceback
 
-# Function definitions remain the same
 def calculate_bleu(translations, references):
     return sacrebleu.corpus_bleu(translations, [references]).score
 
@@ -18,30 +17,44 @@ def calculate_bertscore(translations, references, lang):
     P, R, F1 = bert_score(translations, references, lang=lang)
     return F1.mean().item()
 
-# Streamlit app
+def tokenize_text(text, lang_code):
+    if lang_code == "zh":
+        return ' '.join(jieba.cut(text))
+    # For other languages, we'll use a simple space-based tokenization
+    # This might not be ideal for all languages, but it's a start
+    return ' '.join(text.split())
+
 st.title("Machine Translation Quality Evaluation")
 st.write("Input the translated text and the reference translation to compute BLEU, TER, CHRF, and BERTScore metrics.")
 
-# Language selection and input fields remain the same
-# ...
+languages = {
+    "English": "en", "Chinese": "zh", "French": "fr", "German": "de", "Spanish": "es",
+    "Russian": "ru", "Japanese": "ja", "Korean": "ko", "Arabic": "ar", "Italian": "it",
+    "Dutch": "nl", "Portuguese": "pt", "Turkish": "tr", "Polish": "pl", "Czech": "cs",
+    "Swedish": "sv", "Danish": "da", "Finnish": "fi", "Greek": "el", "Hungarian": "hu",
+    "Indonesian": "id", "Norwegian": "no", "Romanian": "ro", "Thai": "th", "Vietnamese": "vi",
+    "Hebrew": "he", "Hindi": "hi", "Bengali": "bn", "Tamil": "ta", "Urdu": "ur", "Other": "other"
+}
+
+source_lang = st.selectbox("Select Source Language", list(languages.keys()))
+target_lang = st.selectbox("Select Target Language", list(languages.keys()))
+
+source_lang_code = st.text_input("Enter Source Language Code (ISO 639-1):", value=languages[source_lang]) if source_lang == "Other" else languages[source_lang]
+target_lang_code = st.text_input("Enter Target Language Code (ISO 639-1):", value=languages[target_lang]) if target_lang == "Other" else languages[target_lang]
+
+translation_input = st.text_area("Translated Text", height=200)
+reference_input = st.text_area("Reference Translation", height=200)
 
-# Evaluate button
 if st.button("Evaluate"):
     if translation_input and reference_input:
         try:
-            translations = [translation_input.strip()]
-            references = [reference_input.strip()]
+            translations = [tokenize_text(translation_input.strip(), target_lang_code)]
+            references = [tokenize_text(reference_input.strip(), target_lang_code)]
 
-            st.write("Debug: Inputs received")
+            st.write("Debug: Inputs received and tokenized")
             st.write(f"Translation: {translations}")
             st.write(f"Reference: {references}")
 
-            # Handle tokenization if necessary (e.g., for Chinese)
-            if source_lang_code == "zh" or target_lang_code == "zh":
-                translations = [' '.join(jieba.cut(text)) for text in translations]
-                references = [' '.join(jieba.cut(text)) for text in references]
-                st.write("Debug: Chinese tokenization applied")
-
             st.write("Debug: Calculating scores...")
 
             bleu_score = calculate_bleu(translations, references)