from __future__ import annotations
from huggingface_hub import HfFileSystem
import gradio as gr
import random
import os
import bcrypt
from typing import Optional, Any
from loguru import logger
from IPython.display import display, Markdown
from datetime import datetime, timezone
import yaml
import uuid
from glob import glob
from evidence_seeker import EvidenceSeeker
from evidence_seeker import RetrievalConfig
from evidence_seeker import ConfirmationAnalyzerConfig
from evidence_seeker import ClaimPreprocessingConfig
def get_sources(documents, confirmation_by_document) -> str | None:
grouped = {}
for doc in documents:
if doc["metadata"]["file_name"] not in grouped.keys():
grouped[doc["metadata"]["file_name"]] = {"author": doc["metadata"]["author"],
"url": doc["metadata"]["url"],
"title": doc["metadata"]["title"].replace("{","").replace("}",""),
"texts": [{"original_text": doc["metadata"]["original_text"],
"conf": confirmation_by_document[doc["uid"]],
"full_text": doc["text"]}]}
else:
grouped[doc["metadata"]["file_name"]]["texts"].append({"original_text": doc["metadata"]["original_text"], "conf": confirmation_by_document[doc["uid"]], "full_text": doc["text"]})
t = []
for doc in grouped.keys():
grouped[doc]["texts"] = sorted(grouped[doc]["texts"], key=lambda item: item["conf"], reverse=True)
t.append(f"- {grouped[doc]['author']}: *{grouped[doc]['title']}* ([Link]({grouped[doc]['url']})):")
for text in grouped[doc]["texts"]:
short = f'"{text["original_text"].strip().replace("\n"," ").replace('"',"'")}" **[{round(text["conf"],5)}]**'
detailed = f'" {text["full_text"].strip().replace("\n"," ").replace('"',"'")}"'
part = f" - {short}\n \n Mehr Details
\n\n - {detailed}\n\n "
t.append(part)
if len(t) == 0:
return None
else:
return "\n\n".join(t) + "\n\n"
def describe_result(claim: str, results: list) -> str:
preamble_template = (
'## EvidenceSeeker Results\n\n'
'### Input\n\n'
'**Submitted claim:** {claim}\n\n'
'### Results\n\n'
)
result_template = (
'**Clarified claim:** {text} [_{statement_type}_]\n\n'
'**Status**: {verbalized_confirmation}\n\n'
'|Metric|Value|\n'
'|:---|---:|\n'
'|Average confirmation|{average_confirmation:.3f}|\n'
'|Evidential divergence|{evidential_uncertainty:.3f}|\n'
'|Width of evidential base|{n_evidence}|\n\n'
)
markdown = []
markdown.append(preamble_template.format(claim=claim))
for claim_dict in results:
rdict = claim_dict.copy()
rdict["statement_type"] = rdict["statement_type"].value
markdown.append(result_template.format(**claim_dict))
s = get_sources(rdict["documents"],rdict["confirmation_by_document"])
if s:
s = "\n\n\n\nUsed documents
\n\n" + s + " \n\n"
markdown.append(s)
return "\n".join(markdown)
def setup(index_path: Optional[str] = None) -> EvidenceSeeker:
# TODO: Load configs from YAML files. (should, perhaps, be saved under ./config/)
retrieval_config = RetrievalConfig(
embed_backend_type="huggingface_inference_api",
embed_model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
embed_base_url="https://router.huggingface.co/hf-inference/models/sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
api_key_name="hf_debatelab_inference_provider",
hub_key_name="hf_evse_data",
bill_to="DebateLabKIT",
#index_hub_path="DebateLabKIT/apuz-index-es",
# Can be used if indexed is stored locally.
index_persist_path="../evidence-seeker/TMP/APUZ/storage",
)
confirmation_analysis_config = ConfirmationAnalyzerConfig(
timeout=1200,
used_model_key="together.ai"
)
preprocessing_config = ClaimPreprocessingConfig(
timeout=1200,
used_model_key="together.ai"
)
evidence_seeker = EvidenceSeeker(retrieval_config=retrieval_config,confirmation_analysis_config=confirmation_analysis_config,preprocessing_config=preprocessing_config)
return evidence_seeker
def draw_example(examples : list[str]) -> str:
random.shuffle(examples)
return examples[0]
def check_password(input_password: str, hash : str) -> bool:
return bcrypt.checkpw(input_password.encode(), hash.encode())
def initialize_result_state(pipeline):
if pipeline:
last_result = {"pipeline": {}}
preprocessing_config = pipeline.preprocessor.workflow.config.model_dump()
preprocessor = preprocessing_config["models"][preprocessing_config["used_model_key"]]
confirmation_analysis_config = pipeline.analyzer.workflow.config.model_dump()
analyzer = confirmation_analysis_config["models"][confirmation_analysis_config["used_model_key"]]
last_result["pipeline"]["models"] = {
"preprocessing": {"model": preprocessor["model"], "backend_type": preprocessor["backend_type"], "max_tokens": preprocessor["max_tokens"], "temperature": preprocessor["temperature"]},
"confirmation_analysis": {"model": analyzer["model"], "backend_type": analyzer["backend_type"], "max_tokens": analyzer["max_tokens"], "temperature": analyzer["temperature"]},
"retrieval": {"model": pipeline.retriever.embed_model_name, "backend_type": pipeline.retriever.embed_backend_type, "embed_batch_size": pipeline.retriever.embed_batch_size, "top_k": pipeline.retriever.similarity_top_k}
}
index_path = "https://huggingface.co/datasets/" + pipeline.retriever.index_hub_path if pipeline.retriever.index_hub_path else "local"
last_result["pipeline"]["index"] = index_path
last_result["pipeline"]["ignored_statement_types"] = [t for t in pipeline.retriever.ignore_statement_types]
else:
last_result = {'pipeline': None}
return gr.State(last_result)
def reactivate(check_btn, statement):
if statement.strip() != "":
check_btn = gr.Button(visible=True,interactive=True)
pos_btn = gr.Button(visible=True,interactive=True)
neg_btn = gr.Button(visible=True,interactive=True)
feedback = gr.Markdown(visible=True)
return feedback, check_btn, pos_btn, neg_btn
def deactivate():
check_btn = gr.Button(interactive=False)
pos_btn = gr.Button(interactive=False, variant="secondary", visible=False)
neg_btn = gr.Button(interactive=False, variant="secondary", visible=False)
feedback = gr.Markdown(visible=False)
return check_btn, pos_btn, neg_btn, feedback
def log_feedback(feedback_btn, last_result):
choice = "positive" if feedback_btn == "👍" else "negative"
last_result["feedback"] = None if last_result["feedback"] == choice else choice
logger.log('INFO',f"{last_result["feedback"]} feedback on results")
return gr.Button(variant="primary" if last_result["feedback"] else "secondary"), gr.Button(variant="secondary")
def log_result(last_result : dict[str, Any]):
global RUNS_ON_SPACES, HF_BASE, LOCAL_BASE, RESULT_DIR
fn = f"evseek_request_{last_result['request_id']}.yaml"
subdir = _current_subdir("daily")
if RUNS_ON_SPACES:
if "EVIDENCE_SEEKER_APPDATA" in os.environ.keys():
fs = HfFileSystem(token=os.environ["EVIDENCE_SEEKER_APPDATA"])
else:
fs = HfFileSystem()
try:
files = fs.glob("/".join([HF_BASE, RESULT_DIR, "**", fn]))
assert len(files) < 2
if len(files) == 0:
filepath = "/".join([HF_BASE, RESULT_DIR, subdir, fn])
fs.makedirs("/".join([HF_BASE, RESULT_DIR, subdir]), exist_ok=True)
else:
filepath = files[0]
except FileNotFoundError:
filepath = "/".join([HF_BASE, RESULT_DIR, subdir, fn])
fs.makedirs("/".join([HF_BASE, RESULT_DIR, subdir]), exist_ok=True)
with fs.open(filepath, encoding="utf-8", mode="w") as f:
yaml.dump(last_result, f, allow_unicode=True, default_flow_style=False, encoding='utf-8')
else:
files = glob("/".join([LOCAL_BASE, RESULT_DIR, "**", fn]), recursive=True)
assert len(files) < 2
if len(files) == 0:
filepath = "/".join([LOCAL_BASE, RESULT_DIR, subdir, fn])
os.makedirs("/".join([LOCAL_BASE, RESULT_DIR, subdir]), exist_ok=True)
else:
filepath = files[0]
with open(filepath, encoding="utf-8", mode="w") as f:
yaml.dump(last_result, f, allow_unicode=True, default_flow_style=False, encoding='utf-8')
def _current_subdir(subdirectory_construction : str) -> str:
now = datetime.now()
if subdirectory_construction == "monthly":
subdirectory_path = now.strftime("y%Y_m%m")
elif subdirectory_construction == "weekly":
year, week, _ = now.isocalendar()
subdirectory_path = f"y{year}_w{week}"
elif subdirectory_construction == "yearly":
subdirectory_path = now.strftime("y%Y")
elif subdirectory_construction == "daily":
subdirectory_path = now.strftime("%Y_%m_%d")
else:
subdirectory_path = ""
return subdirectory_path
EXAMPLE_INPUTS = [
"Die Osterweiterung hat die EU-Institutionen nachhaltig geschwächt.",
"In den knapp 70 Jahren seit ihrer Gründung hat es in der Bundeswehr immer wieder rechtsextremistische Vorfälle gegeben.",
"In der Bundeswehr gibt es keinen politischen Extremismus.",
"Die BRICS-Staaten sorgen für eine Veränderung der westlich geprägten Weltordnung.",
"Die Genfer Konventionen sind oft hinter ihrem Anspruch, die Zivilbevölkerung zu schützen, zurückgeblieben.",
"Die Anzahl hybrider Kriege hat zugenommen.",
"Es ist für Deutschland wirtschaftlich ein Nachteil, dass viele Frauen in Teilzeit arbeiten.",
"Premier Modi hat Putin als seinen Freund bezeichnet.",
"Eine Minderheit der deutschen Bevölkerung befürwortet einen autoritären deutschen Staat.",
]
HF_BASE = 'datasets/DebateLabKIT/evidence-seeker-appdata'
RESULT_DIR = 'result_data'
LOCAL_BASE = '../TMP'
if "RUNS_ON_SPACES" not in os.environ.keys():
RUNS_ON_SPACES = False
# TODO: Remove - path to .env file should be set in the configt
# (if it is set, it will be loaded automatically)
from dotenv import load_dotenv
load_dotenv()
logger.info("Gradioapp runs locally.")
else:
RUNS_ON_SPACES = os.environ["RUNS_ON_SPACES"] == "True"
logger.info("Gradioapp runs on 🤗.")
EVIDENCE_SEEKER = setup()
with gr.Blocks(title="EvidenceSeeker") as demo:
valid = gr.State(value=not RUNS_ON_SPACES)
examples = gr.State(EXAMPLE_INPUTS)
last_result = initialize_result_state(EVIDENCE_SEEKER)
pos_btn = gr.Button(value="👍",visible=False,interactive=False,render=False)
neg_btn = gr.Button(value="👎",visible=False,interactive=False,render=False)
feedback = gr.Markdown(value="Wie zufrieden bist du mit der Antwort?",render=False,visible=False)
@gr.render(inputs=[valid])
def renderApp(v):
if v:
gr.Markdown("""
# 🕵️ EvidenceSeeker DemoApp
Gib eine Aussage in das Textfeld ein und lass sie durch den EvidenceSeeker prüfen:
""", key="intro")
with gr.Row() as row:
statement = gr.Textbox(label="Zu prüfende Aussage:", interactive=True, value="",scale=10,lines=3,key="statement")
with gr.Column(scale=1):
example_btn = gr.Button("Zufälliges Beispiel",key="example_btn")
check_btn = gr.Button("Prüfe Aussage", interactive=False,key="submit_btn")
results = gr.Markdown(value="", min_height=80, key="results")
with gr.Column():
feedback.render()
with gr.Row():
pos_btn.render()
neg_btn.render()
async def check(statement : str, last_result):
last_result["request_time"] = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
last_result["request"] = statement
last_result["claims"] = []
last_result["request_id"] = str(uuid.uuid4())
last_result["feedback"] = None
if valid is False:
md = ""
elif EVIDENCE_SEEKER is None:
md = f"Kein EvidenceSeeker initialisiert um '{statement}' zu prüfen."
else:
result = await EVIDENCE_SEEKER(statement)
last_result["claims"] = [{"text": claim["text"],"negation":claim["negation"],"statement_type": claim["statement_type"].value,"average_confirmation": claim["average_confirmation"],"verbalized_confirmation": claim["verbalized_confirmation"]} for claim in result]
md = describe_result(statement, result)
logger.info(last_result)
return md, last_result
example_btn.click(fn=draw_example, inputs=examples, outputs=statement)
check_btn.click(deactivate, None, [check_btn, pos_btn, neg_btn, feedback]).then((lambda: "### Working..."), None, results).then(check, [statement, last_result], [results, last_result]).then(log_result, last_result, None).then(reactivate, [check_btn, statement], [feedback, check_btn, pos_btn, neg_btn])
statement.change(lambda s: (gr.Button("Prüfe Aussage", interactive=False,key="submit_btn") if s.strip() == "" else gr.Button("Prüfe Aussage", interactive=True,key="submit_btn")), statement, [check_btn])
pos_btn.click(log_feedback, [pos_btn, last_result], [pos_btn, neg_btn]).then(log_result, last_result, None)
neg_btn.click(log_feedback, [neg_btn, last_result], [neg_btn, pos_btn]).then(log_result, last_result, None)
else:
with gr.Column() as req:
gr.Markdown("""
# 🕵️ EvidenceSeeker DemoApp
"""
)
box = gr.Textbox(label="Please enter password for access", autofocus=True, type="password", submit_btn=True)
res = gr.Markdown(value="")
def auth(pw : str, valid : bool):
output = ""
b = gr.Textbox(value="")
if "APP_HASH" not in os.environ:
output = "Something went wrong on our end :-("
elif not check_password(pw, os.environ["APP_HASH"]):
output = "Wrong password. Try again."
else:
output = "Continuing..."
valid = True
return output, valid, b, req
box.submit(auth,inputs=[box,valid],outputs=[res,valid, box, req])
demo.launch()