From e6224a7800034a46211a5bedb92c27c63cc727bc Mon Sep 17 00:00:00 2001 From: Sahana Yadnakudige Subramanya Date: Wed, 15 Apr 2026 14:35:19 +0200 Subject: [PATCH 01/47] added optimization code --- .../optimization/baseline_optimization.py | 302 ++++++++++++++++ .../optimization/contrastive_optimization.py | 333 +++++++++++++++++ .../generate_contrastive_answers.py | 125 +++++++ learn2rag/optimization/readme.md | 0 .../optimization/self_judge_optimization.py | 340 ++++++++++++++++++ 5 files changed, 1100 insertions(+) create mode 100644 learn2rag/optimization/baseline_optimization.py create mode 100644 learn2rag/optimization/contrastive_optimization.py create mode 100644 learn2rag/optimization/generate_contrastive_answers.py create mode 100644 learn2rag/optimization/readme.md create mode 100644 learn2rag/optimization/self_judge_optimization.py diff --git a/learn2rag/optimization/baseline_optimization.py b/learn2rag/optimization/baseline_optimization.py new file mode 100644 index 0000000..ea5b0c4 --- /dev/null +++ b/learn2rag/optimization/baseline_optimization.py @@ -0,0 +1,302 @@ +""" +RAG Pipeline Optimization with BERTScore evaluation. +""" + +import argparse +import json +import logging +import pathlib +import time +import copy + +import numpy as np +from bert_score import score as bert_score +from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation +from smac import HyperparameterOptimizationFacade, Scenario + +from learn2rag.evaluation.tools import read_dataset_qa +from learn2rag.pipeline.config import opt_config +import learn2rag.pipeline.search +import learn2rag.pipeline.generate + + +DATASET_CONFIG = { + "WikiEval": { + "subdirectory": "", + "split": "train", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "rag-mini-bioasq": { + "subdirectory": "question-answer-passages", + "split": "test", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "hotpot_qa": { # Not being used + "subdirectory": "distractor", + "split": "validation", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "repliqa": { # Not being used + "subdirectory": "repliqa_4", + "split": None, + "question_field": "question", + "answer_field": "long_answer", + "id_field": "question_id", + }, +} + +PROMPT_MAP = { + "default": ( + "# Role and Objective\nYou will act as a smart AI chatbot that answers " + "questions only by using the content from the provided information list.\n\n" + "# Instructions\n- Respond in the language of the question.\n" + "- Answer clear and concise.\n- Only use the provided information.\n" + "- NEVER use your general knowledge.\n\n" + "# Information:\n{context}" + ), + "concise": ( + "Answer the question using ONLY the provided information. " + "Be concise and direct. If the information does not contain the answer, say so.\n\n" + "Information:\n{context}" + ), + "detailed": ( + "You are a knowledgeable assistant. Using ONLY the provided information below, " + "answer the question thoroughly. Cite your sources. " + "If the information is insufficient, state that clearly.\n\n" + "Information:\n{context}" + ), +} + + +def run_pipeline(question, user_config, working_config): + t0 = time.time() + docs = learn2rag.pipeline.search.search(question, user_config, working_config) + search_time = time.time() - t0 + + t0 = time.time() + answer = learn2rag.pipeline.generate.generate(question, docs, working_config) + gen_time = time.time() - t0 + + doc_list = docs.points if hasattr(docs, "points") else docs + context = "" + if doc_list: + context = "\n\n".join([ + f"Source: {d.payload.get('path', 'unknown')}\nContent: {d.payload.get('content', '')}" + for d in doc_list + ]) + return answer, context[:3000], search_time, gen_time + + +def objective(config, seed, questions, dataset_name, state, answers_dir): + state["trial_count"] += 1 + tid = state["trial_count"] + cfg = dict(config) + logging.info(f"Trial {tid}: {cfg}") + + wcfg = copy.deepcopy(opt_config) + wcfg["top_k"] = cfg["top_k"] + wcfg["chunk_size"] = cfg["chunk_size"] + wcfg["chunk_overlap"] = cfg["chunk_overlap"] + wcfg["prompt"] = PROMPT_MAP[cfg["prompt_template"]] + ucfg = {"file_path": None, "collection_name": dataset_name, + "imported_documents_file_path": None, "llm": None} + + predictions, goldens = [], [] + qa_pairs = [] + t_start = time.time() + t_search, t_gen = 0.0, 0.0 + + for idx, q in enumerate(questions): + if not q["question"]: + continue + try: + answer, context, st, gt = run_pipeline(q["question"], ucfg, wcfg) + t_search += st + t_gen += gt + predictions.append(answer) + goldens.append(q["ground_truth"]) + qa_pairs.append({ + "id": q["id"], + "question": q["question"], + "golden_answer": q["ground_truth"], + "generated_answer": answer, + "retrieved_context": context, + }) + except Exception as e: + logging.warning(f"Trial {tid}, q{idx} failed: {e}") + predictions.append("") + goldens.append(q["ground_truth"]) + qa_pairs.append({ + "id": q["id"], + "question": q["question"], + "golden_answer": q["ground_truth"], + "generated_answer": "", + "retrieved_context": "", + }) + + if not predictions: + return 1.0 + + t_score = time.time() + _, _, F1_gold = bert_score(predictions, goldens, lang="en", verbose=False, rescale_with_baseline=True) + scoring_time = time.time() - t_score + + # objective function + bert_gold = [max(0.0, f.item()) for f in F1_gold] + avg_bert_gold = np.mean(bert_gold) + cost = 1.0 - avg_bert_gold + total_time = time.time() - t_start + + trial_answers = { + "trial_id": tid, + "config": cfg, + "cost": float(cost), + "avg_bertscore_golden": float(avg_bert_gold), + "qa_pairs": qa_pairs, + } + answers_file = answers_dir / f"trial_{tid}_answers.json" + with open(answers_file, "w") as f: + json.dump(trial_answers, f, indent=2, default=str) + + state["best_cost"] = min(state["best_cost"], cost) + state["convergence"].append({"trial": tid, "cost": float(cost), "best_cost": float(state["best_cost"])}) + state["history"].append({ + "trial_id": tid, "config": cfg, + "avg_bertscore_golden": float(avg_bert_gold), + "cost": float(cost), "time_s": round(total_time, 2), + "search_s": round(t_search, 2), "gen_s": round(t_gen, 2), + "scoring_s": round(scoring_time, 2), + }) + + logging.info( + f"Trial {tid}: bertscore_golden={avg_bert_gold:.4f} cost={cost:.4f} " + f"time={total_time:.1f}s (search={t_search:.1f} gen={t_gen:.1f} score={scoring_time:.1f})" + ) + return float(cost) + + +def param_importance(smac, output_path): + params = list(smac.scenario.configspace.keys()) + configs, costs = [], [] + for key, val in smac.runhistory.items(): + configs.append(dict(smac.runhistory.get_config(key.config_id))) + costs.append(val.cost) + if len(configs) < 3: + return {} + + raw = {} + for p in params: + groups = {} + for c, cost in zip(configs, np.array(costs)): + groups.setdefault(str(c[p]), []).append(cost) + means = [np.mean(g) for g in groups.values()] + raw[p] = float(np.var(means)) if len(means) > 1 else 0.0 + + total = sum(raw.values()) + imp = {p: round(v / total, 4) for p, v in raw.items()} if total > 0 else raw + ranking = sorted(imp, key=imp.get, reverse=True) + result = {"method": "variance_based", "ranking": ranking, "individual": imp} + with open(output_path / "parameter_importance.json", "w") as f: + json.dump(result, f, indent=2) + return result + + +def run(dataset_name, max_questions, n_trials, output_dir): + if dataset_name not in DATASET_CONFIG: + raise ValueError( + f"Unknown dataset: {dataset_name}. " + f"Available: {list(DATASET_CONFIG.keys())}" + ) + + dcfg = DATASET_CONFIG[dataset_name] + out = pathlib.Path(output_dir) / dataset_name + out.mkdir(parents=True, exist_ok=True) + + answers_dir = out / "trial_answers" + answers_dir.mkdir(parents=True, exist_ok=True) + + qa = read_dataset_qa(dataset_name, dcfg["subdirectory"], dcfg["split"]) + if max_questions: + qa = qa.select(range(min(max_questions, len(qa)))) + + questions = [] + for i, r in enumerate(qa): + questions.append({ + "question": r.get(dcfg["question_field"], ""), + "ground_truth": r.get(dcfg["answer_field"], ""), + "id": r.get(dcfg["id_field"], str(i)), + }) + logging.info(f"Loaded {len(questions)} questions from {dataset_name}") + + cs = ConfigurationSpace(seed=42) + cs.add([Integer("top_k", (1, 20), default=4), + Integer("chunk_size", (200, 4000), default=2000), + Integer("chunk_overlap", (0, 500), default=200), + Categorical("prompt_template", ["default", "concise", "detailed"], default="default")]) + cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) + + scenario = Scenario(configspace=cs, deterministic=True, n_trials=n_trials, + walltime_limit=7200, seed=42, output_directory=out / "smac_output") + + state = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + smac = HyperparameterOptimizationFacade( + scenario=scenario, + target_function=lambda config, seed=0: objective( + config, seed, questions, dataset_name, state, answers_dir + ), + ) + + t0 = time.time() + incumbent = smac.optimize() + total_time = time.time() - t0 + + importance = param_importance(smac, out) + + with open(out / "optimization_results.json", "w") as f: + json.dump({ + "best_config": dict(incumbent), + "run_history": state["history"], + "convergence": state["convergence"], + "parameter_importance": importance, + "total_time_s": round(total_time, 2), + "dataset": dataset_name, + "metric": "bertscore_golden", + "answers_dir": str(answers_dir), + }, f, indent=2, default=str) + + logging.info(f"Done in {total_time:.0f}s") + logging.info(f"Trial answers saved to {answers_dir}") + return incumbent, state["history"], importance + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, default="WikiEval", + choices=list(DATASET_CONFIG.keys())) + parser.add_argument("--max_questions", type=int, default=50) + parser.add_argument("--n_trials", type=int, default=10) + parser.add_argument("--output_dir", type=str, default="optimization_results_baseline") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + force=True, + ) + incumbent, history, importance = run( + args.dataset, args.max_questions, args.n_trials, args.output_dir, + ) + + best = min(history, key=lambda x: x["cost"]) + print(f"\nBest config: {dict(incumbent)}") + print(f"BERTScore (golden): {best['avg_bertscore_golden']:.4f}") + if importance: + print(f"\nParameter importance:") + for i, p in enumerate(importance["ranking"], 1): + print(f" {i}. {p}: {importance['individual'][p]:.4f}") \ No newline at end of file diff --git a/learn2rag/optimization/contrastive_optimization.py b/learn2rag/optimization/contrastive_optimization.py new file mode 100644 index 0000000..91a5d50 --- /dev/null +++ b/learn2rag/optimization/contrastive_optimization.py @@ -0,0 +1,333 @@ +""" +RAG Pipeline Optimization with contrastive evaluation. +""" + +import argparse +import json +import logging +import pathlib +import time +import copy + +import numpy as np +from bert_score import score as bert_score +from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation +from smac import HyperparameterOptimizationFacade, Scenario + +from learn2rag.evaluation.tools import read_dataset_qa +from learn2rag.pipeline.config import opt_config +import learn2rag.pipeline.search +import learn2rag.pipeline.generate + + +DATASET_CONFIG = { + "WikiEval": { + "subdirectory": "", + "split": "train", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "rag-mini-bioasq": { + "subdirectory": "question-answer-passages", + "split": "test", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "hotpot_qa": { # Not being used + "subdirectory": "distractor", + "split": "validation", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "repliqa": { # Not being used + "subdirectory": "repliqa_4", + "split": None, + "question_field": "question", + "answer_field": "long_answer", + "id_field": "question_id", + }, +} + +PROMPT_MAP = { + "default": ( + "# Role and Objective\nYou will act as a smart AI chatbot that answers " + "questions only by using the content from the provided information list.\n\n" + "# Instructions\n- Respond in the language of the question.\n" + "- Answer clear and concise.\n- Only use the provided information.\n" + "- NEVER use your general knowledge.\n\n" + "# Information:\n{context}" + ), + "concise": ( + "Answer the question using ONLY the provided information. " + "Be concise and direct. If the information does not contain the answer, say so.\n\n" + "Information:\n{context}" + ), + "detailed": ( + "You are a knowledgeable assistant. Using ONLY the provided information below, " + "answer the question thoroughly. Cite your sources. " + "If the information is insufficient, state that clearly.\n\n" + "Information:\n{context}" + ), +} + + +def load_false_answers(contrastive_file): + with open(contrastive_file) as f: + data = json.load(f) + return {item["question"]: item["false_answer"] for item in data} + + +def run_pipeline(question, user_config, working_config): + t0 = time.time() + docs = learn2rag.pipeline.search.search(question, user_config, working_config) + search_time = time.time() - t0 + + t0 = time.time() + answer = learn2rag.pipeline.generate.generate(question, docs, working_config) + gen_time = time.time() - t0 + + doc_list = docs.points if hasattr(docs, "points") else docs + context = "" + if doc_list: + context = "\n\n".join([ + f"Source: {d.payload.get('path', 'unknown')}\nContent: {d.payload.get('content', '')}" + for d in doc_list + ]) + return answer, context[:3000], search_time, gen_time + + +def objective(config, seed, questions, dataset_name, false_map, state, answers_dir): + state["trial_count"] += 1 + tid = state["trial_count"] + cfg = dict(config) + logging.info(f"Trial {tid}: {cfg}") + + wcfg = copy.deepcopy(opt_config) + wcfg["top_k"] = cfg["top_k"] + wcfg["chunk_size"] = cfg["chunk_size"] + wcfg["chunk_overlap"] = cfg["chunk_overlap"] + wcfg["prompt"] = PROMPT_MAP[cfg["prompt_template"]] + ucfg = {"file_path": None, "collection_name": dataset_name, + "imported_documents_file_path": None, "llm": None} + + predictions, goldens, falses = [], [], [] + qa_pairs = [] + t_start = time.time() + t_search, t_gen = 0.0, 0.0 + + for idx, q in enumerate(questions): + if not q["question"]: + continue + false_ans = false_map.get(q["question"], "") + if not false_ans: + continue + try: + answer, context, st, gt = run_pipeline(q["question"], ucfg, wcfg) + t_search += st + t_gen += gt + predictions.append(answer) + goldens.append(q["ground_truth"]) + falses.append(false_ans) + qa_pairs.append({ + "id": q["id"], + "question": q["question"], + "golden_answer": q["ground_truth"], + "generated_answer": answer, + "retrieved_context": context, + }) + except Exception as e: + logging.warning(f"Trial {tid}, q{idx} failed: {e}") + predictions.append("") + goldens.append(q["ground_truth"]) + falses.append(false_ans) + qa_pairs.append({ + "id": q["id"], + "question": q["question"], + "golden_answer": q["ground_truth"], + "generated_answer": "", + "retrieved_context": "", + }) + + if not predictions: + return 1.0 + + t_score = time.time() + _, _, F1_gold = bert_score(predictions, goldens, lang="en", verbose=False, rescale_with_baseline=True) + _, _, F1_false = bert_score(predictions, falses, lang="en", verbose=False, rescale_with_baseline=True) + scoring_time = time.time() - t_score + + bert_gold = [max(0.0, f.item()) for f in F1_gold] + bert_false = [max(0.0, f.item()) for f in F1_false] + + ratios = [] + # objective function + for bg, bf in zip(bert_gold, bert_false): + denom = bg + bf + ratios.append(bg / denom if denom > 0 else 0.5) + + avg_ratio = np.mean(ratios) + avg_bert_gold = np.mean(bert_gold) + avg_bert_false = np.mean(bert_false) + cost = 1.0 - avg_ratio + total_time = time.time() - t_start + + trial_answers = { + "trial_id": tid, + "config": cfg, + "cost": float(cost), + "avg_ratio": float(avg_ratio), + "qa_pairs": qa_pairs, + } + answers_file = answers_dir / f"trial_{tid}_answers.json" + with open(answers_file, "w") as f: + json.dump(trial_answers, f, indent=2, default=str) + + state["best_cost"] = min(state["best_cost"], cost) + state["convergence"].append({"trial": tid, "cost": float(cost), "best_cost": float(state["best_cost"])}) + state["history"].append({ + "trial_id": tid, "config": cfg, + "avg_ratio": float(avg_ratio), + "avg_bertscore_golden": float(avg_bert_gold), + "avg_bertscore_false": float(avg_bert_false), + "cost": float(cost), "time_s": round(total_time, 2), + "search_s": round(t_search, 2), "gen_s": round(t_gen, 2), + "scoring_s": round(scoring_time, 2), + }) + + logging.info( + f"Trial {tid}: ratio={avg_ratio:.4f} [gold={avg_bert_gold:.4f} false={avg_bert_false:.4f}] " + f"time={total_time:.1f}s (search={t_search:.1f} gen={t_gen:.1f} score={scoring_time:.1f})" + ) + return float(cost) + + +def param_importance(smac, output_path): + params = list(smac.scenario.configspace.keys()) + configs, costs = [], [] + for key, val in smac.runhistory.items(): + configs.append(dict(smac.runhistory.get_config(key.config_id))) + costs.append(val.cost) + if len(configs) < 3: + return {} + + raw = {} + for p in params: + groups = {} + for c, cost in zip(configs, np.array(costs)): + groups.setdefault(str(c[p]), []).append(cost) + means = [np.mean(g) for g in groups.values()] + raw[p] = float(np.var(means)) if len(means) > 1 else 0.0 + + total = sum(raw.values()) + imp = {p: round(v / total, 4) for p, v in raw.items()} if total > 0 else raw + ranking = sorted(imp, key=imp.get, reverse=True) + result = {"method": "variance_based", "ranking": ranking, "individual": imp} + with open(output_path / "parameter_importance.json", "w") as f: + json.dump(result, f, indent=2) + return result + + +def run(dataset_name, max_questions, n_trials, output_dir, contrastive_dir): + if dataset_name not in DATASET_CONFIG: + raise ValueError( + f"Unknown dataset: {dataset_name}. " + f"Available: {list(DATASET_CONFIG.keys())}" + ) + + dcfg = DATASET_CONFIG[dataset_name] + contrastive_file = pathlib.Path(contrastive_dir) / f"contrastive_answers_{dataset_name}.json" + out = pathlib.Path(output_dir) / dataset_name + out.mkdir(parents=True, exist_ok=True) + + answers_dir = out / "trial_answers" + answers_dir.mkdir(parents=True, exist_ok=True) + + false_map = load_false_answers(contrastive_file) + logging.info(f"Loaded {len(false_map)} false answers from {contrastive_file}") + + qa = read_dataset_qa(dataset_name, dcfg["subdirectory"], dcfg["split"]) + if max_questions: + qa = qa.select(range(min(max_questions, len(qa)))) + + questions = [] + for i, r in enumerate(qa): + questions.append({ + "question": r.get(dcfg["question_field"], ""), + "ground_truth": r.get(dcfg["answer_field"], ""), + "id": r.get(dcfg["id_field"], str(i)), + }) + logging.info(f"Loaded {len(questions)} questions from {dataset_name}") + + cs = ConfigurationSpace(seed=42) + cs.add([Integer("top_k", (1, 20), default=4), + Integer("chunk_size", (200, 4000), default=2000), + Integer("chunk_overlap", (0, 500), default=200), + Categorical("prompt_template", ["default", "concise", "detailed"], default="default")]) + cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) + + scenario = Scenario(configspace=cs, deterministic=True, n_trials=n_trials, + walltime_limit=7200, seed=42, output_directory=out / "smac_output") + + state = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + smac = HyperparameterOptimizationFacade( + scenario=scenario, + target_function=lambda config, seed=0: objective( + config, seed, questions, dataset_name, false_map, state, answers_dir + ), + ) + + t0 = time.time() + incumbent = smac.optimize() + total_time = time.time() - t0 + + importance = param_importance(smac, out) + + with open(out / "optimization_results.json", "w") as f: + json.dump({ + "best_config": dict(incumbent), + "run_history": state["history"], + "convergence": state["convergence"], + "parameter_importance": importance, + "total_time_s": round(total_time, 2), + "dataset": dataset_name, + "metric": "bertscore_ratio(golden, false)", + "answers_dir": str(answers_dir), + }, f, indent=2, default=str) + + logging.info(f"Done in {total_time:.0f}s") + logging.info(f"Trial answers saved to {answers_dir}") + return incumbent, state["history"], importance + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, default="WikiEval", + choices=list(DATASET_CONFIG.keys())) + parser.add_argument("--max_questions", type=int, default=50) + parser.add_argument("--n_trials", type=int, default=10) + parser.add_argument("--output_dir", type=str, default="optimization_results_contrastive") + parser.add_argument("--contrastive_dir", type=str, default="contrastive_answers") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + force=True, + ) + incumbent, history, importance = run( + args.dataset, args.max_questions, args.n_trials, + args.output_dir, args.contrastive_dir, + ) + + best = min(history, key=lambda x: x["cost"]) + print(f"\nBest config: {dict(incumbent)}") + print(f"Ratio score: {best['avg_ratio']:.4f}") + print(f" BERTScore vs golden: {best['avg_bertscore_golden']:.4f}") + print(f" BERTScore vs false: {best['avg_bertscore_false']:.4f}") + if importance: + print(f"\nParameter importance:") + for i, p in enumerate(importance["ranking"], 1): + print(f" {i}. {p}: {importance['individual'][p]:.4f}") \ No newline at end of file diff --git a/learn2rag/optimization/generate_contrastive_answers.py b/learn2rag/optimization/generate_contrastive_answers.py new file mode 100644 index 0000000..feb233f --- /dev/null +++ b/learn2rag/optimization/generate_contrastive_answers.py @@ -0,0 +1,125 @@ +""" +Generate contrastive (false) answers for RAG evaluation. +""" + +import argparse +import json +import logging +import pathlib +import time + +from learn2rag.pipeline.llm import llm as learn2rag_llm +from learn2rag.evaluation.tools import read_dataset_qa +from langchain_core.messages import HumanMessage + + +DATASET_CONFIG = { + "WikiEval": { + "subdirectory": "", + "split": "train", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "rag-mini-bioasq": { + "subdirectory": "question-answer-passages", + "split": "test", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "hotpot_qa": { # Not being used + "subdirectory": "distractor", + "split": "validation", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "repliqa": { # Not being used + "subdirectory": "repliqa_4", + "split": None, + "question_field": "question", + "answer_field": "long_answer", + "id_field": "question_id", + }, +} + + +def generate(prompt): + response = learn2rag_llm.invoke([HumanMessage(content=prompt)]) + return response.content.strip() + + +def generate_false_answer(question): + return generate( + f"Answer the given question in an incorrect manner.\n\n" + f"question: {question}" + ) + +def run(dataset_name, max_questions=50, output_dir="contrastive_answers"): + if dataset_name not in DATASET_CONFIG: + raise ValueError( + f"Unknown dataset: {dataset_name}. " + f"Available: {list(DATASET_CONFIG.keys())}" + ) + + cfg = DATASET_CONFIG[dataset_name] + output_path = pathlib.Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + qa = read_dataset_qa(dataset_name, cfg["subdirectory"], cfg["split"]) + if max_questions: + qa = qa.select(range(min(max_questions, len(qa)))) + + logging.info(f"Dataset: {dataset_name}, questions: {len(qa)}") + + results = [] + total_start = time.time() + + for idx, item in enumerate(qa): + question = item.get(cfg["question_field"], "") + golden = item.get(cfg["answer_field"], "") + qid = item.get(cfg["id_field"], str(idx)) + + if not question: + continue + + print(f"[{idx+1}/{len(qa)}] {question[:60]}...") + + t0 = time.time() + false_answer = generate_false_answer(question) + elapsed = time.time() - t0 + + results.append({ + "id": qid, + "question": question, + "golden_answer": golden, + "false_answer": false_answer, + "generation_time_s": round(elapsed, 2), + }) + + total_time = time.time() - total_start + + out_file = output_path / f"contrastive_answers_{dataset_name}.json" + with open(out_file, "w") as f: + json.dump(results, f, indent=2) + + print(f"\nGenerated false answers for {len(results)} questions in {total_time:.0f}s") + print(f"Saved to {out_file}") + return results + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, default="WikiEval", + choices=list(DATASET_CONFIG.keys())) + parser.add_argument("--max_questions", type=int, default=50) + parser.add_argument("--output_dir", type=str, default="contrastive_answers") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + force=True, + ) + run(args.dataset, args.max_questions, args.output_dir) \ No newline at end of file diff --git a/learn2rag/optimization/readme.md b/learn2rag/optimization/readme.md new file mode 100644 index 0000000..e69de29 diff --git a/learn2rag/optimization/self_judge_optimization.py b/learn2rag/optimization/self_judge_optimization.py new file mode 100644 index 0000000..93a9dc0 --- /dev/null +++ b/learn2rag/optimization/self_judge_optimization.py @@ -0,0 +1,340 @@ +""" +RAG pipeline optimization with self-judge using answer relevance only. +""" + +import argparse +import json +import logging +import pathlib +import time +import copy +import re + +import numpy as np +from sentence_transformers import SentenceTransformer +from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation +from smac import HyperparameterOptimizationFacade, Scenario + +from learn2rag.evaluation.tools import read_dataset_qa +from learn2rag.pipeline.config import opt_config +from learn2rag.pipeline.llm import llm as learn2rag_llm +from langchain_core.messages import HumanMessage +import learn2rag.pipeline.search +import learn2rag.pipeline.generate + + +DATASET_CONFIG = { + "WikiEval": { + "subdirectory": "", + "split": "train", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "rag-mini-bioasq": { + "subdirectory": "question-answer-passages", + "split": "test", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "hotpot_qa": { # Not being used + "subdirectory": "distractor", + "split": "validation", + "question_field": "question", + "answer_field": "answer", + "id_field": "id", + }, + "repliqa": { # Not being used + "subdirectory": "repliqa_4", + "split": None, + "question_field": "question", + "answer_field": "long_answer", + "id_field": "question_id", + }, +} + +PROMPT_MAP = { + "default": ( + "# Role and Objective\nYou will act as a smart AI chatbot that answers " + "questions only by using the content from the provided information list.\n\n" + "# Instructions\n- Respond in the language of the question.\n" + "- Answer clear and concise.\n- Only use the provided information.\n" + "- NEVER use your general knowledge.\n\n" + "# Information:\n{context}" + ), + "concise": ( + "Answer the question using ONLY the provided information. " + "Be concise and direct. If the information does not contain the answer, say so.\n\n" + "Information:\n{context}" + ), + "detailed": ( + "You are a knowledgeable assistant. Using ONLY the provided information below, " + "answer the question thoroughly. Cite your sources. " + "If the information is insufficient, state that clearly.\n\n" + "Information:\n{context}" + ), +} + +encoder = SentenceTransformer("all-MiniLM-L6-v2") + + +def call_llm(prompt, max_tokens=512): + response = learn2rag_llm.invoke([HumanMessage(content=prompt)], max_tokens=max_tokens) + return response.content.strip() + + +def answer_relevance(question, answer): + if not question or not answer: + return 0.0 + + generated_qs = [] + for _ in range(3): + text = call_llm( + f"Generate a question for the given answer.\n\nanswer: {answer}", + max_tokens=128, + ) + first_line = text.split("\n")[0].strip() + first_line = re.sub(r"^\d+[\.\)]\s*", "", first_line).strip() + if len(first_line) > 10: + generated_qs.append(first_line) + + if not generated_qs: + return 0.0 + + q_emb = encoder.encode([question])[0] + gen_embs = encoder.encode(generated_qs) + sims = [] + for ge in gen_embs: + cos = np.dot(q_emb, ge) / (np.linalg.norm(q_emb) * np.linalg.norm(ge) + 1e-8) + sims.append(max(0.0, float(cos))) + + return np.mean(sims) + + +def run_pipeline(question, user_config, working_config): + t0 = time.time() + docs = learn2rag.pipeline.search.search(question, user_config, working_config) + search_time = time.time() - t0 + + t0 = time.time() + answer = learn2rag.pipeline.generate.generate(question, docs, working_config) + gen_time = time.time() - t0 + + doc_list = docs.points if hasattr(docs, "points") else docs + context = "" + if doc_list: + context = "\n\n".join([ + f"Source: {d.payload.get('path', 'unknown')}\nContent: {d.payload.get('content', '')}" + for d in doc_list + ]) + return answer, context[:3000], search_time, gen_time + + +def objective(config, seed, questions, dataset_name, state, answers_dir): + state["trial_count"] += 1 + tid = state["trial_count"] + cfg = dict(config) + logging.info(f"Trial {tid}: {cfg}") + + wcfg = copy.deepcopy(opt_config) + wcfg["top_k"] = cfg["top_k"] + wcfg["chunk_size"] = cfg["chunk_size"] + wcfg["chunk_overlap"] = cfg["chunk_overlap"] + wcfg["prompt"] = PROMPT_MAP[cfg["prompt_template"]] + ucfg = {"file_path": None, "collection_name": dataset_name, + "imported_documents_file_path": None, "llm": None} + + rel_scores = [] + qa_pairs = [] + t_start = time.time() + t_search, t_gen, t_judge = 0.0, 0.0, 0.0 + + for idx, q in enumerate(questions): + if not q["question"]: + continue + try: + answer, context, st, gt = run_pipeline(q["question"], ucfg, wcfg) + t_search += st + t_gen += gt + + tj = time.time() + rel = answer_relevance(q["question"], answer) + rel_scores.append(rel) + t_judge += time.time() - tj + + qa_pairs.append({ + "id": q["id"], + "question": q["question"], + "ground_truth": q["ground_truth"], + "generated_answer": answer, + "retrieved_context": context, + "answer_relevance": float(rel), + }) + except Exception as e: + logging.warning(f"Trial {tid}, q{idx} failed: {e}") + rel_scores.append(0.0) + qa_pairs.append({ + "id": q["id"], + "question": q["question"], + "ground_truth": q["ground_truth"], + "generated_answer": "", + "retrieved_context": "", + "answer_relevance": 0.0, + }) + + if not rel_scores: + return 1.0 + + # Objective function + avg_rel = float(np.mean(rel_scores)) + cost = 1.0 - avg_rel + total_time = time.time() - t_start + + trial_data = { + "trial_id": tid, + "config": cfg, + "avg_answer_relevance": avg_rel, + "cost": float(cost), + "time_s": round(total_time, 2), + "qa_pairs": qa_pairs, + } + with open(answers_dir / f"trial_{tid}_answers.json", "w") as f: + json.dump(trial_data, f, indent=2) + + state["best_cost"] = min(state["best_cost"], cost) + state["convergence"].append({"trial": tid, "cost": float(cost), "best_cost": float(state["best_cost"])}) + state["history"].append({ + "trial_id": tid, "config": cfg, + "avg_answer_relevance": avg_rel, + "cost": float(cost), + "time_s": round(total_time, 2), + "search_s": round(t_search, 2), "gen_s": round(t_gen, 2), "judge_s": round(t_judge, 2), + }) + + logging.info( + f"Trial {tid}: answer_relevance={avg_rel:.4f} cost={cost:.4f} " + f"time={total_time:.1f}s (search={t_search:.1f} gen={t_gen:.1f} judge={t_judge:.1f})" + ) + return float(cost) + + +def param_importance(smac, output_path): + params = list(smac.scenario.configspace.keys()) + configs, costs = [], [] + for key, val in smac.runhistory.items(): + configs.append(dict(smac.runhistory.get_config(key.config_id))) + costs.append(val.cost) + if len(configs) < 3: + return {} + + raw = {} + for p in params: + groups = {} + for c, cost in zip(configs, np.array(costs)): + groups.setdefault(str(c[p]), []).append(cost) + means = [np.mean(g) for g in groups.values()] + raw[p] = float(np.var(means)) if len(means) > 1 else 0.0 + + total = sum(raw.values()) + imp = {p: round(v / total, 4) for p, v in raw.items()} if total > 0 else raw + ranking = sorted(imp, key=imp.get, reverse=True) + result = {"method": "variance_based", "ranking": ranking, "individual": imp} + with open(output_path / "parameter_importance.json", "w") as f: + json.dump(result, f, indent=2) + return result + + +def run(dataset_name, max_questions, n_trials, output_dir): + if dataset_name not in DATASET_CONFIG: + raise ValueError( + f"Unknown dataset: {dataset_name}. " + f"Available: {list(DATASET_CONFIG.keys())}" + ) + + dcfg = DATASET_CONFIG[dataset_name] + out = pathlib.Path(output_dir) / dataset_name + out.mkdir(parents=True, exist_ok=True) + answers_dir = out / "trial_answers" + answers_dir.mkdir(parents=True, exist_ok=True) + + qa = read_dataset_qa(dataset_name, dcfg["subdirectory"], dcfg["split"]) + if max_questions: + qa = qa.select(range(min(max_questions, len(qa)))) + + questions = [] + for i, r in enumerate(qa): + questions.append({ + "question": r.get(dcfg["question_field"], ""), + "ground_truth": r.get(dcfg["answer_field"], ""), + "id": r.get(dcfg["id_field"], str(i)), + }) + logging.info(f"Loaded {len(questions)} questions from {dataset_name}") + + cs = ConfigurationSpace(seed=42) + cs.add([Integer("top_k", (1, 20), default=4), + Integer("chunk_size", (200, 4000), default=2000), + Integer("chunk_overlap", (0, 500), default=200), + Categorical("prompt_template", ["default", "concise", "detailed"], default="default")]) + cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) + + scenario = Scenario(configspace=cs, deterministic=True, n_trials=n_trials, + walltime_limit=36000, seed=42, output_directory=out / "smac_output") + + state = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + smac = HyperparameterOptimizationFacade( + scenario=scenario, + target_function=lambda config, seed=0: objective( + config, seed, questions, dataset_name, state, answers_dir + ), + ) + + t0 = time.time() + incumbent = smac.optimize() + total_time = time.time() - t0 + + importance = param_importance(smac, out) + + with open(out / "optimization_results.json", "w") as f: + json.dump({ + "best_config": dict(incumbent), + "run_history": state["history"], + "convergence": state["convergence"], + "parameter_importance": importance, + "total_time_s": round(total_time, 2), + "dataset": dataset_name, + "generator": "gemma-3-27b-it", + "judge": "gemma-3-27b-it (self-judge)", + "metrics": ["answer_relevance"], + "metric_source": "Es et al. (2023) RAGAS arXiv:2309.15217v2", + }, f, indent=2, default=str) + + logging.info(f"Done in {total_time:.0f}s") + return incumbent, state["history"], importance + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, default="WikiEval", + choices=list(DATASET_CONFIG.keys())) + parser.add_argument("--max_questions", type=int, default=50) + parser.add_argument("--n_trials", type=int, default=10) + parser.add_argument("--output_dir", type=str, default="optimization_results_selfjudge") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + force=True, + ) + incumbent, history, importance = run( + args.dataset, args.max_questions, args.n_trials, args.output_dir, + ) + + best = min(history, key=lambda x: x["cost"]) + print(f"\nBest config: {dict(incumbent)}") + print(f"Answer Relevance: {best['avg_answer_relevance']:.4f}") + if importance: + print(f"\nParameter importance:") + for i, p in enumerate(importance["ranking"], 1): + print(f" {i}. {p}: {importance['individual'][p]:.4f}") \ No newline at end of file From 0f8f87f504a72d53e90247b6b0f66a1518bff64c Mon Sep 17 00:00:00 2001 From: farshad68 Date: Fri, 24 Apr 2026 11:46:54 +0200 Subject: [PATCH 02/47] first integrated version , RAG throw an exception --- learn2rag/evaluation/tools.py | 1 + learn2rag/pipeline/config.py | 53 +++++++++++++++++++++++++++++++++-- learn2rag/ui/__init__.py | 6 ++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/learn2rag/evaluation/tools.py b/learn2rag/evaluation/tools.py index b91d0d7..3da8ffe 100644 --- a/learn2rag/evaluation/tools.py +++ b/learn2rag/evaluation/tools.py @@ -80,6 +80,7 @@ def ingest_dataset_documents(dataset_name: str) -> None: def read_dataset_qa(dataset_name: str, subdirectory: str, split: str | None=None) -> Any: logging.debug(f'{dataset_name=}') dataset_work_dir = pathlib.Path('./datasets') / dataset_name + dataset_work_dir = dataset_work_dir.resolve() dataset_dict = datasets.load_from_disk(dataset_work_dir / 'source' / subdirectory) return dataset_dict[split] if split is not None else dataset_dict diff --git a/learn2rag/pipeline/config.py b/learn2rag/pipeline/config.py index 1433172..772f351 100644 --- a/learn2rag/pipeline/config.py +++ b/learn2rag/pipeline/config.py @@ -1,5 +1,12 @@ import json import os +import logging +import collections.abc +from typing import Any + + + +opt_config: dict[str, Any] = {} with open(os.environ.get("PIPELINE_USER_CONFIG", "learn2rag/pipeline/user_config.json"), "r") as file: user_config = json.load(file) @@ -7,5 +14,47 @@ with open(os.environ.get("IMPORTER_CONFIG", "learn2rag/importer/config/config.json"), "r") as file: importer_config = json.load(file) -with open(os.environ.get("PIPELINE_OPT_CONFIG", "learn2rag/pipeline/opt_config.json"), "r") as file: - opt_config = json.load(file) +def refresh_configs(): + global opt_config + logging.info(f"Refreshing configs...") + with open(os.environ.get("PIPELINE_OPT_CONFIG", "learn2rag/pipeline/opt_config.json"), "r") as file: + base_data = json.load(file) + + final_data = base_data + + patch_path = os.environ.get("PIPELINE_OPT_PATCH_CONFIG", "learn2rag/pipeline/opt_patch_config.json") + if os.path.exists(patch_path): + logging.info(f"Patch found! Layering {patch_path} onto defaults.") + try: + with open(patch_path, "r") as f: + patch_data = json.load(f) + # Overlay the patch onto our final_data + final_data = deep_update(base_data, patch_data) + except json.JSONDecodeError: + logging.error(f"Patch file at {patch_path} is corrupted. Skipping overlay.") + else: + # This is the "Fallback" behavior + logging.warning(f"No patch file found at {patch_path}. System will run on default settings.") + logging.info(f"Refreshing configs done {final_data}") + opt_config.clear() + opt_config.update(final_data) + + logging.info(f"FINAL OPT_CONFIG STRUCTURE: {json.dumps(opt_config, indent=2)}") + + +def deep_update(base_dict, overrides): + """Recursively merges overrides into base_dict.""" + for key, value in overrides.items(): + if isinstance(value, collections.abc.Mapping): + if key in base_dict and not isinstance(base_dict[key], collections.abc.Mapping): + logging.error(f"Type Mismatch at '{key}': expected dict, got {type(base_dict[key])}") + base_dict[key] = {} + base_dict[key] = deep_update(base_dict.get(key, {}), value) + else: + logging.info(f"Overriding '{key}': {base_dict.get(key)} -> {value}") + base_dict[key] = value + return base_dict + + +# Initial load on module import +refresh_configs() \ No newline at end of file diff --git a/learn2rag/ui/__init__.py b/learn2rag/ui/__init__.py index 0678800..c63fb05 100644 --- a/learn2rag/ui/__init__.py +++ b/learn2rag/ui/__init__.py @@ -28,8 +28,8 @@ from learn2rag.compose import Project import learn2rag.data import learn2rag.pipeline.llm - -from datetime import datetime # <-- ADD THIS +from learn2rag.pipeline.config import refresh_configs +from datetime import datetime logging.getLogger().addHandler(flask.logging.default_handler) @@ -388,7 +388,7 @@ def pipeline_create() -> 'str | werkzeug.wrappers.response.Response': def start_pipeline(name: str, pipeline: dict[str, Any], template_name: str) -> None: has_ssl = bool(app.config.get("TLS")) - + refresh_configs() url = urllib.parse.urlparse(request.base_url) assert url.scheme From 1f1a562224aff93c54b61eaeccb5c0c101e235bb Mon Sep 17 00:00:00 2001 From: farshad68 Date: Fri, 24 Apr 2026 12:06:28 +0200 Subject: [PATCH 03/47] add new dependencies --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index feb8e7f..e53ed41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -188,6 +188,10 @@ dependencies = [ "python-pptx (>=1.0.2,<2.0.0)", "msoffcrypto-tool (>=6.0.0,<7.0.0)", "ebooklib (>=0.20,<0.21)", + "bert-score==0.3.13", + "ConfigSpace==1.2.2", + "scikit-learn==1.8.0", + "smac (>=2.3.1)" ] description = "" license = "MIT" From 4f9a4a0e78f4f5c0253f2941dc71e83e3497f7b9 Mon Sep 17 00:00:00 2001 From: farshad68 Date: Wed, 29 Apr 2026 10:53:09 +0200 Subject: [PATCH 04/47] add more logs and fallback --- learn2rag/pipeline/config.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/learn2rag/pipeline/config.py b/learn2rag/pipeline/config.py index 772f351..1bb9a9e 100644 --- a/learn2rag/pipeline/config.py +++ b/learn2rag/pipeline/config.py @@ -8,17 +8,33 @@ opt_config: dict[str, Any] = {} -with open(os.environ.get("PIPELINE_USER_CONFIG", "learn2rag/pipeline/user_config.json"), "r") as file: - user_config = json.load(file) +user_config_path = os.environ.get("PIPELINE_USER_CONFIG", "learn2rag/pipeline/user_config.json") +try: + with open(user_config_path , "r") as file: + user_config = json.load(file) +except: + logging.error(f"User Config file not found at {user_config_path}. Using default settings.") + user_config = {} -with open(os.environ.get("IMPORTER_CONFIG", "learn2rag/importer/config/config.json"), "r") as file: - importer_config = json.load(file) + +importer_config_path = os.environ.get("IMPORTER_CONFIG", "learn2rag/importer/config/config.json") +try: + with open(importer_config_path, "r") as file: + importer_config = json.load(file) +except: + logging.error(f"Importer Config file not found at {importer_config_path}. Using default settings.") + importer_config = {} def refresh_configs(): global opt_config logging.info(f"Refreshing configs...") - with open(os.environ.get("PIPELINE_OPT_CONFIG", "learn2rag/pipeline/opt_config.json"), "r") as file: - base_data = json.load(file) + pipeline_config_path = os.environ.get("PIPELINE_OPT_CONFIG", "learn2rag/pipeline/opt_config.json") + try: + with open(pipeline_config_path, "r") as file: + base_data = json.load(file) + except: + logging.error(f"Pipeline Config file not found at {pipeline_config_path}. Using default settings.") + base_data = {} final_data = base_data From bb37d532dbae2ca25841b66b196c1bb93d73bb2e Mon Sep 17 00:00:00 2001 From: farshad68 Date: Wed, 29 Apr 2026 16:28:37 +0200 Subject: [PATCH 05/47] make config simple - now we just rewrite the base config - no patching --- learn2rag/evaluation/tools.py | 2 - learn2rag/optimization/__init__.py | 62 ++++++++++ learn2rag/pipeline/config.py | 77 ++---------- learn2rag/ui/__init__.py | 3 +- .../compose/pipelines/optimization.yml | 114 ++++++++++++++++++ 5 files changed, 184 insertions(+), 74 deletions(-) create mode 100644 learn2rag/optimization/__init__.py create mode 100644 learn2rag/ui/templates/compose/pipelines/optimization.yml diff --git a/learn2rag/evaluation/tools.py b/learn2rag/evaluation/tools.py index 3da8ffe..d221fa5 100644 --- a/learn2rag/evaluation/tools.py +++ b/learn2rag/evaluation/tools.py @@ -78,9 +78,7 @@ def ingest_dataset_documents(dataset_name: str) -> None: def read_dataset_qa(dataset_name: str, subdirectory: str, split: str | None=None) -> Any: - logging.debug(f'{dataset_name=}') dataset_work_dir = pathlib.Path('./datasets') / dataset_name - dataset_work_dir = dataset_work_dir.resolve() dataset_dict = datasets.load_from_disk(dataset_work_dir / 'source' / subdirectory) return dataset_dict[split] if split is not None else dataset_dict diff --git a/learn2rag/optimization/__init__.py b/learn2rag/optimization/__init__.py new file mode 100644 index 0000000..24703c2 --- /dev/null +++ b/learn2rag/optimization/__init__.py @@ -0,0 +1,62 @@ +import logging +import pathlib +import os +import json +import collections.abc +import copy +from os import mkdir + +from . import baseline_optimization +#TODO : now we need to copy the dataset to here manually it should consider in installation maybe ! +# {storage_path}/datasets/WikiEval/ +def main() -> None: + logging.error("optimization is started") + dataset_name = "WikiEval" + + output_dir = pathlib.Path("./optimization/output/") + output_dir.mkdir(parents=True, exist_ok=True) + + baseline_optimization.run(dataset_name,10,10,output_dir) + logging.error("optimization is done") + results_path = output_dir /dataset_name/ "optimization_results.json" + logging.info(f"save optimized results here : {results_path}") + if not results_path.exists(): + logging.error(f"Optimization results not found at {results_path}") + return + + with open(results_path, "r") as f: + full_results = json.load(f) + + best_config = full_results.get("best_config", {}) + + target_config_path = os.environ.get("PIPELINE_OPT_CONFIG", "learn2rag/pipeline/opt_config.json") + + existing_config = {} + if os.path.exists(target_config_path): + try: + with open(target_config_path, "r", encoding="utf-8") as f: + existing_config = json.load(f) + except json.JSONDecodeError: + logging.error(f"Existing config at {target_config_path} is corrupted. It will be overwritten.") + + updated_config = deep_update(existing_config, best_config) + + pathlib.Path(target_config_path).parent.mkdir(parents=True, exist_ok=True) + with open(target_config_path, "w", encoding="utf-8") as f: + json.dump(updated_config, f, indent=4) + + logging.info(f"Successfully updated opt_config at: {target_config_path}") + +def deep_update(base_dict, overrides): + """Recursively merges overrides into a copy of base_dict, returning the new dict.""" + result = copy.deepcopy(base_dict) + for key, value in overrides.items(): + if isinstance(value, collections.abc.Mapping): + if key in result and not isinstance(result[key], collections.abc.Mapping): + logging.error(f"Type Mismatch at '{key}': expected dict, got {type(result[key])}") + result[key] = {} + result[key] = deep_update(result.get(key, {}), value) + else: + logging.info(f"Overriding '{key}': {result.get(key)} -> {value}") + result[key] = value + return result diff --git a/learn2rag/pipeline/config.py b/learn2rag/pipeline/config.py index 1bb9a9e..b0677fb 100644 --- a/learn2rag/pipeline/config.py +++ b/learn2rag/pipeline/config.py @@ -1,76 +1,13 @@ import json import os import logging -import collections.abc -from typing import Any +with open(os.environ.get("PIPELINE_USER_CONFIG", "learn2rag/pipeline/user_config.json"), "r") as file: + user_config = json.load(file) +with open(os.environ.get("IMPORTER_CONFIG", "learn2rag/importer/config/config.json"), "r") as file: + importer_config = json.load(file) -opt_config: dict[str, Any] = {} - -user_config_path = os.environ.get("PIPELINE_USER_CONFIG", "learn2rag/pipeline/user_config.json") -try: - with open(user_config_path , "r") as file: - user_config = json.load(file) -except: - logging.error(f"User Config file not found at {user_config_path}. Using default settings.") - user_config = {} - - -importer_config_path = os.environ.get("IMPORTER_CONFIG", "learn2rag/importer/config/config.json") -try: - with open(importer_config_path, "r") as file: - importer_config = json.load(file) -except: - logging.error(f"Importer Config file not found at {importer_config_path}. Using default settings.") - importer_config = {} - -def refresh_configs(): - global opt_config - logging.info(f"Refreshing configs...") - pipeline_config_path = os.environ.get("PIPELINE_OPT_CONFIG", "learn2rag/pipeline/opt_config.json") - try: - with open(pipeline_config_path, "r") as file: - base_data = json.load(file) - except: - logging.error(f"Pipeline Config file not found at {pipeline_config_path}. Using default settings.") - base_data = {} - - final_data = base_data - - patch_path = os.environ.get("PIPELINE_OPT_PATCH_CONFIG", "learn2rag/pipeline/opt_patch_config.json") - if os.path.exists(patch_path): - logging.info(f"Patch found! Layering {patch_path} onto defaults.") - try: - with open(patch_path, "r") as f: - patch_data = json.load(f) - # Overlay the patch onto our final_data - final_data = deep_update(base_data, patch_data) - except json.JSONDecodeError: - logging.error(f"Patch file at {patch_path} is corrupted. Skipping overlay.") - else: - # This is the "Fallback" behavior - logging.warning(f"No patch file found at {patch_path}. System will run on default settings.") - logging.info(f"Refreshing configs done {final_data}") - opt_config.clear() - opt_config.update(final_data) - - logging.info(f"FINAL OPT_CONFIG STRUCTURE: {json.dumps(opt_config, indent=2)}") - - -def deep_update(base_dict, overrides): - """Recursively merges overrides into base_dict.""" - for key, value in overrides.items(): - if isinstance(value, collections.abc.Mapping): - if key in base_dict and not isinstance(base_dict[key], collections.abc.Mapping): - logging.error(f"Type Mismatch at '{key}': expected dict, got {type(base_dict[key])}") - base_dict[key] = {} - base_dict[key] = deep_update(base_dict.get(key, {}), value) - else: - logging.info(f"Overriding '{key}': {base_dict.get(key)} -> {value}") - base_dict[key] = value - return base_dict - - -# Initial load on module import -refresh_configs() \ No newline at end of file +with open(os.environ.get("PIPELINE_OPT_CONFIG", "learn2rag/pipeline/opt_config.json"), "r") as file: + opt_config = json.load(file) + logging.info(f"Loaded opt_config:\n{json.dumps(opt_config, indent=4)}") diff --git a/learn2rag/ui/__init__.py b/learn2rag/ui/__init__.py index 2d9739b..068190e 100644 --- a/learn2rag/ui/__init__.py +++ b/learn2rag/ui/__init__.py @@ -29,7 +29,7 @@ from learn2rag.compose import Project import learn2rag.data import learn2rag.pipeline.llm -from learn2rag.pipeline.config import refresh_configs + from datetime import datetime @@ -410,7 +410,6 @@ def pipeline_create() -> 'str | werkzeug.wrappers.response.Response': def start_pipeline(name: str, pipeline: dict[str, Any], template_name: str) -> None: has_ssl = bool(app.config.get("TLS")) - refresh_configs() url = urllib.parse.urlparse(request.base_url) assert url.scheme diff --git a/learn2rag/ui/templates/compose/pipelines/optimization.yml b/learn2rag/ui/templates/compose/pipelines/optimization.yml new file mode 100644 index 0000000..c721f88 --- /dev/null +++ b/learn2rag/ui/templates/compose/pipelines/optimization.yml @@ -0,0 +1,114 @@ +name: optimization +label: Optimize +ports: + # TODO: labels in the interface currently assume a specific port order + - ui + - qdrant_http +files: + - path: '{{storage_path}}/basic_user_config.json' + content: | + { + "collection_name": "learn2rag", + "imported_documents_file_path": "loaded_documents.json", + "llm": "{{language_model.model}}" + } + - path: '{{storage_path}}/qdrant_config.yml' + content: | + log_level: INFO + service: + api_key: '{{qdrant_api_key}}' + grpc_port: null + http_port: '{{ports.qdrant_http}}' + host: '127.0.0.1' + telemetry_disabled: true + - path: '{{storage_path}}/logging_config.yml' + content: | + version: 1 + formatters: + simple: + format: "%(asctime)s %(levelname)-8s %(name)s %(message)s" + colored: + class: colorlog.ColoredFormatter + format: "%(log_color)s%(asctime)s %(levelname)-8s %(name)s %(message)s" + profiling: + format: "%(created)f %(request_id)s %(activity)s %(message)s" + handlers: + display: + class: logging.StreamHandler + level: INFO + formatter: colored + stream: ext://sys.stderr + profiling_display: + class: logging.StreamHandler + formatter: profiling + stream: ext://sys.stderr + profiling_file: + class: logging.FileHandler + formatter: profiling + filename: '{{storage_path}}/logs/profiling.log' + encoding: utf-8 + errors_file: + class: logging.FileHandler + level: ERROR + formatter: simple + filename: '{{storage_path}}/logs/error.log' + encoding: utf-8 + {% if debug_logging %} + debug_file: + class: logging.FileHandler + level: DEBUG + formatter: simple + filename: '{{storage_path}}/logs/debug.log' + encoding: utf-8 + {% endif %} + loggers: + profiling: + handlers: + - profiling_display + - profiling_file + propagate: no + root: + level: DEBUG + handlers: + - display + - errors_file + {% if debug_logging %} + - debug_file + {% endif %} + + - path: '{{storage_path}}/logs/.keep' + content: '' + +services: + qdrant: + working_dir: '{{storage_path}}' + command: + - '{{learn2rag_path}}/services/qdrant/qdrant{% if is_windows %}.exe{% endif %}' + - '--config-path' + - '{{storage_path}}/qdrant_config.yml' + # https://qdrant.tech/documentation/guides/configuration/ + environment: + QDRANT__LOG_LEVEL: 'ERROR' + QDRANT__SERVICE__HOST: '127.0.0.1' + QDRANT__SERVICE__HTTP_PORT: '{{ports.qdrant_http}}' + QDRANT__TELEMETRY_DISABLED: 'true' + main: + working_dir: '{{storage_path}}' + command: + - '{{learn2rag_path}}/configurator{% if is_windows %}.exe{% endif %}' + - 'learn2rag.optimization' + - '--logging-config' + - '{{storage_path}}/logging_config.yml' + environment: + LEARN2RAG_PATH: '{{learn2rag_path}}' + QDRANT__SERVICE__HTTP_PORT: '{{ports.qdrant_http}}' + QDRANT__SERVICE__API_KEY: '{{qdrant_api_key}}' + PIPELINE_USER_CONFIG: '{{storage_path}}/basic_user_config.json' + IMPORTER_CONFIG: '{{storage_path}}/importer_config.json' + PIPELINE_OPT_CONFIG: '{{learn2rag_path}}/learn2rag/pipeline/opt_config.json' + LANGCHAIN_API_KEY: '1' + LANGCHAIN_TRACING_V2: 'false' + LLM_API_TYPE: '{{language_model.api}}' + LLM_API_URL: '{{language_model.url}}' + LLM_API_TOKEN: '{{language_model.token}}' + LLM_API_MODEL: '{{language_model.model}}' From 78bb9aff8f06c063403fb2a9e004a83973decc93 Mon Sep 17 00:00:00 2001 From: farshad68 Date: Mon, 4 May 2026 11:57:50 +0200 Subject: [PATCH 06/47] make dataset names and args not hardcoded in optimization __init__ --- learn2rag/evaluation/tools.py | 1 + learn2rag/optimization/__init__.py | 54 +++++++++++++++++++----------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/learn2rag/evaluation/tools.py b/learn2rag/evaluation/tools.py index d221fa5..b91d0d7 100644 --- a/learn2rag/evaluation/tools.py +++ b/learn2rag/evaluation/tools.py @@ -78,6 +78,7 @@ def ingest_dataset_documents(dataset_name: str) -> None: def read_dataset_qa(dataset_name: str, subdirectory: str, split: str | None=None) -> Any: + logging.debug(f'{dataset_name=}') dataset_work_dir = pathlib.Path('./datasets') / dataset_name dataset_dict = datasets.load_from_disk(dataset_work_dir / 'source' / subdirectory) return dataset_dict[split] if split is not None else dataset_dict diff --git a/learn2rag/optimization/__init__.py b/learn2rag/optimization/__init__.py index 24703c2..84a6ef1 100644 --- a/learn2rag/optimization/__init__.py +++ b/learn2rag/optimization/__init__.py @@ -4,24 +4,42 @@ import json import collections.abc import copy -from os import mkdir +import yaml +import argparse from . import baseline_optimization #TODO : now we need to copy the dataset to here manually it should consider in installation maybe ! # {storage_path}/datasets/WikiEval/ def main() -> None: - logging.error("optimization is started") - dataset_name = "WikiEval" + parser = argparse.ArgumentParser(description="Run RAG baseline optimization.") + parser.add_argument("task", help="Module task name (e.g., learn2rag.optimization)") + parser.add_argument("--logging-config", type=str, help="Path to logging config yml") + parser.add_argument("--dataset", type=str, default="WikiEval") + parser.add_argument("--questions", type=int, default=10) + parser.add_argument("--trials", type=int, default=10) + args, unknown = parser.parse_known_args() + + if args.logging_config and pathlib.Path(args.logging_config).exists(): + with open(args.logging_config, 'r') as f: + config = yaml.safe_load(f) + logging.config.dictConfig(config) + + logger = logging.getLogger(__name__) + logger.info(f"Running task: {args.task}") + + + logging.info(f"Optimization started for {args.dataset}") + #TODO : read or get dataset_name and maxquestions and n_trails output_dir = pathlib.Path("./optimization/output/") output_dir.mkdir(parents=True, exist_ok=True) - baseline_optimization.run(dataset_name,10,10,output_dir) - logging.error("optimization is done") - results_path = output_dir /dataset_name/ "optimization_results.json" + baseline_optimization.run(args.dataset,args.questions,args.trials,output_dir) + logging.info("optimization is done") + results_path = output_dir /args.dataset/ "optimization_results.json" logging.info(f"save optimized results here : {results_path}") if not results_path.exists(): - logging.error(f"Optimization results not found at {results_path}") + logging.warning(f"Optimization results not found at {results_path}") return with open(results_path, "r") as f: @@ -39,7 +57,7 @@ def main() -> None: except json.JSONDecodeError: logging.error(f"Existing config at {target_config_path} is corrupted. It will be overwritten.") - updated_config = deep_update(existing_config, best_config) + updated_config = deep_update(copy.deepcopy(existing_config), best_config) pathlib.Path(target_config_path).parent.mkdir(parents=True, exist_ok=True) with open(target_config_path, "w", encoding="utf-8") as f: @@ -47,16 +65,14 @@ def main() -> None: logging.info(f"Successfully updated opt_config at: {target_config_path}") -def deep_update(base_dict, overrides): - """Recursively merges overrides into a copy of base_dict, returning the new dict.""" - result = copy.deepcopy(base_dict) +def deep_update(source, overrides): + """Recursively updates a dictionary.""" for key, value in overrides.items(): - if isinstance(value, collections.abc.Mapping): - if key in result and not isinstance(result[key], collections.abc.Mapping): - logging.error(f"Type Mismatch at '{key}': expected dict, got {type(result[key])}") - result[key] = {} - result[key] = deep_update(result.get(key, {}), value) + if isinstance(value, collections.abc.Mapping) and key in source: + deep_update(source.get(key, {}), value) else: - logging.info(f"Overriding '{key}': {result.get(key)} -> {value}") - result[key] = value - return result + source[key] = value + return source + +if __name__ == "__main__": + main() \ No newline at end of file From a5ee3507dfdb4613bf2ed6255f750da614a6579e Mon Sep 17 00:00:00 2001 From: farshad68 Date: Mon, 4 May 2026 12:00:25 +0200 Subject: [PATCH 07/47] chore: update uv.lock after dependency changes --- uv.lock | 243 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) diff --git a/uv.lock b/uv.lock index ba2dd60..1244a31 100644 --- a/uv.lock +++ b/uv.lock @@ -208,6 +208,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "bert-score" +version = "0.3.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "requests" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/93/2c97a85cbb66a8a256a13176e11c9c4508074e2341299fe75ee955c81eff/bert_score-0.3.13.tar.gz", hash = "sha256:8ffe5838eac8cdd988b8b1a896af7f49071188c8c011a1ed160d71a9899a2ba4", size = 48621, upload-time = "2023-02-20T21:07:29.477Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/8c/bc5457de4c004b1a623b31f7bc8d0375fb699b7d67df11879098b4b7b7c8/bert_score-0.3.13-py3-none-any.whl", hash = "sha256:bbbb4c7fcdaa46d7681aff49f37f96faa09ed74e1b150e659bdc6b58a66989b9", size = 61135, upload-time = "2023-02-20T21:07:27.226Z" }, +] + [[package]] name = "blinker" version = "1.9.0" @@ -350,6 +369,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/8a/c4bb04426d608be4a3171efa2e233d2c59a5c8937850c10d098e126df18e/cloudpathlib-0.23.0-py3-none-any.whl", hash = "sha256:8520b3b01468fee77de37ab5d50b1b524ea6b4a8731c35d1b7407ac0cd716002", size = 62755, upload-time = "2025-10-07T22:47:54.905Z" }, ] +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -401,6 +429,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8d/e4/d66708bdf0d92fb4d49b22cdff4b10cec38aca5dcd7e81d909bb55c65cd7/confection-1.3.3-py3-none-any.whl", hash = "sha256:b9fef9ee84b237ef4611ec3eb5797b70e13063e6310ad9f15536373f5e313c82", size = 35902, upload-time = "2026-03-24T18:45:22.664Z" }, ] +[[package]] +name = "configspace" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, + { name = "numpy" }, + { name = "pyparsing" }, + { name = "scipy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/51/7a63132fdc3a71eea66d9f81b451a61a38631af0a73bdcda1ec784cea32e/configspace-1.2.2.tar.gz", hash = "sha256:b5cc981c145ef9632104d230acf963c2f68fe5a375983946e941c2eec5dd08e4", size = 134158, upload-time = "2025-12-19T12:52:39.628Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/67/01bf8795064d2132d7244341908f75bb2ea87dc114364cd8c5b20c606a8f/configspace-1.2.2-py3-none-any.whl", hash = "sha256:9dd5090fab08fbfc0fa8a66affb53e991e9df65ae20959fcee8e544bb3c0a09d", size = 117584, upload-time = "2025-12-19T12:52:37.545Z" }, +] + [[package]] name = "contourpy" version = "1.3.3" @@ -502,6 +546,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/3f/35701c13e1fc7b0895198c8b20068c569a841e0daf8e0b14d1dc0816b28f/cymem-2.0.13-cp313-cp313t-win_arm64.whl", hash = "sha256:042e8611ef862c34a97b13241f5d0da86d58aca3cecc45c533496678e75c5a1f", size = 38964, upload-time = "2025-11-14T14:58:02.87Z" }, ] +[[package]] +name = "dask" +version = "2026.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "cloudpickle" }, + { name = "fsspec" }, + { name = "packaging" }, + { name = "partd" }, + { name = "pyyaml" }, + { name = "toolz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d7/2a/5d8cc1579590af86576dde890254440e478c7174b93a02095ecfc2e6ba38/dask-2026.3.0.tar.gz", hash = "sha256:f7d96c8274e8a900d217c1ff6ea8d1bbf0b4c2c21e74a409644498d925eb8f85", size = 11000710, upload-time = "2026-03-18T07:10:14.945Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/f3/00bb1e867fba351e2d784170955713bee200c43ea306c59f30bd7e748192/dask-2026.3.0-py3-none-any.whl", hash = "sha256:be614b9242b0b38288060fb2d7696125946469c98a1c30e174883fd199e0428d", size = 1485630, upload-time = "2026-03-18T07:10:12.832Z" }, +] + +[package.optional-dependencies] +distributed = [ + { name = "distributed" }, +] + +[[package]] +name = "dask-jobqueue" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dask" }, + { name = "distributed" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/2a/751bb5ba28533476b1c667eac158e5aa1840c416fd61d425328ef48a894a/dask_jobqueue-0.9.0.tar.gz", hash = "sha256:494ef64b7bb3848c7d72ed334c288030caca6a09dca54cfaa3f395f4ba7f5c47", size = 56475, upload-time = "2024-08-22T09:25:21.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/77/606f138bf70b14865842b3ec9a58dc1ba97153f466e5876fe4ced980f91f/dask_jobqueue-0.9.0-py2.py3-none-any.whl", hash = "sha256:253dfc4f0b8722201a08e05b841859dfeea1f6698ff21eff0d9370e5aa8ae20f", size = 52037, upload-time = "2024-08-22T09:25:20.106Z" }, +] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -583,6 +663,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] +[[package]] +name = "distributed" +version = "2026.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "cloudpickle" }, + { name = "dask" }, + { name = "jinja2" }, + { name = "locket" }, + { name = "msgpack" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "sortedcontainers" }, + { name = "tblib" }, + { name = "toolz" }, + { name = "tornado" }, + { name = "urllib3" }, + { name = "zict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/84/3e98a2d61493d201f253743091cbd4f099f502f957a9a33bb9ce5b8d5416/distributed-2026.3.0.tar.gz", hash = "sha256:4a8fc6102fededfbaae45288501276da2297a054d74eb6589f01b087c7f95c26", size = 2103970, upload-time = "2026-03-18T07:10:23.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/77/2ff7aefc09cf1306a81cd7a46af34f80ebefef81a2e8329b94b58ad813ae/distributed-2026.3.0-py3-none-any.whl", hash = "sha256:52518f4b3e6795e87b442e8f57788ba1ddc750c62d0835669c85927280d38f07", size = 1009769, upload-time = "2026-03-18T07:10:21.241Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -621,6 +727,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/13/563119fe0af82aca5a3b89399c435953072c39515c2e818eb82793955c3b/effdet-0.4.1-py3-none-any.whl", hash = "sha256:10889a226228d515c948e3fcf811e64c0d78d7aa94823a300045653b9c284cb7", size = 112513, upload-time = "2023-05-21T22:17:58.47Z" }, ] +[[package]] +name = "emcee" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/53/1045ee878cb24281387079f8ee4f0ade1622c6aae1ed1fd91a53e4fa5b19/emcee-3.1.6.tar.gz", hash = "sha256:11af4daf6ab8f9ca69681e3c29054665db7bbd87fd4eb8e437d2c3a1248c637d", size = 2871117, upload-time = "2024-04-19T10:03:19.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/ef/2196b9bf88ffa1bde45853c72df021fbd07a8fa91a0f59a22d14a050dc04/emcee-3.1.6-py2.py3-none-any.whl", hash = "sha256:f2d63752023bdccf744461450e512a5b417ae7d28f18e12acd76a33de87580cb", size = 47351, upload-time = "2024-04-19T10:03:17.522Z" }, +] + [[package]] name = "emoji" version = "2.15.0" @@ -1675,6 +1793,7 @@ dependencies = [ { name = "azure-identity" }, { name = "backoff" }, { name = "beautifulsoup4" }, + { name = "bert-score" }, { name = "build" }, { name = "cachetools" }, { name = "certifi" }, @@ -1684,6 +1803,7 @@ dependencies = [ { name = "colorama" }, { name = "coloredlogs" }, { name = "colorlog" }, + { name = "configspace" }, { name = "contourpy" }, { name = "cryptography" }, { name = "cycler" }, @@ -1808,11 +1928,13 @@ dependencies = [ { name = "requests-toolbelt" }, { name = "rsa" }, { name = "safetensors" }, + { name = "scikit-learn" }, { name = "scipy" }, { name = "sentence-transformers" }, { name = "setuptools" }, { name = "simsimd" }, { name = "six" }, + { name = "smac" }, { name = "sniffio" }, { name = "soupsieve" }, { name = "sqlalchemy" }, @@ -1863,6 +1985,7 @@ requires-dist = [ { name = "azure-identity", specifier = ">=1.25.1,<2.0.0" }, { name = "backoff", specifier = "==2.2.1" }, { name = "beautifulsoup4", specifier = ">=4.14.3,<5.0.0" }, + { name = "bert-score", specifier = "==0.3.13" }, { name = "build", specifier = "==1.3.0" }, { name = "cachetools", specifier = "==5.5.2" }, { name = "certifi", specifier = "==2025.8.3" }, @@ -1872,6 +1995,7 @@ requires-dist = [ { name = "colorama", specifier = "==0.4.6" }, { name = "coloredlogs", specifier = "==15.0.1" }, { name = "colorlog", specifier = ">=6.9.0,<7.0.0" }, + { name = "configspace", specifier = "==1.2.2" }, { name = "contourpy", specifier = "==1.3.3" }, { name = "cryptography", specifier = "==45.0.6" }, { name = "cycler", specifier = "==0.12.1" }, @@ -1997,11 +2121,13 @@ requires-dist = [ { name = "requests-toolbelt", specifier = "==1.0.0" }, { name = "rsa", specifier = "==4.9.1" }, { name = "safetensors", specifier = "==0.6.2" }, + { name = "scikit-learn", specifier = "==1.8.0" }, { name = "scipy", specifier = "==1.16.1" }, { name = "sentence-transformers", specifier = ">=5.0.0,<6.0.0" }, { name = "setuptools", specifier = "==80.9.0" }, { name = "simsimd", specifier = ">=6.5.0,<7.0.0" }, { name = "six", specifier = "==1.17.0" }, + { name = "smac", specifier = ">=2.3.1" }, { name = "sniffio", specifier = "==1.3.1" }, { name = "soupsieve", specifier = "==2.7" }, { name = "sqlalchemy", specifier = "==2.0.42" }, @@ -2074,6 +2200,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/a7/d526ae86708cea531935ae777b6dbcabe7db52718e6401e0fb9c5edea80e/llvmlite-0.46.0-cp313-cp313-win_amd64.whl", hash = "sha256:67438fd30e12349ebb054d86a5a1a57fd5e87d264d2451bcfafbbbaa25b82a35", size = 38138941, upload-time = "2025-12-08T18:15:22.536Z" }, ] +[[package]] +name = "locket" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/83/97b29fe05cb6ae28d2dbd30b81e2e402a3eed5f460c26e9eaa5895ceacf5/locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632", size = 4350, upload-time = "2022-04-20T22:04:44.312Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/bc/83e112abc66cd466c6b83f99118035867cecd41802f8d044638aa78a106e/locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3", size = 4398, upload-time = "2022-04-20T22:04:42.23Z" }, +] + [[package]] name = "lxml" version = "6.0.2" @@ -2341,6 +2476,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/bf/dd36e4a6d1cff3f2d30f03e2479cd38210e32d4715bb6a9f0e2737f13604/microsoft_kiota_serialization_text-1.9.10-py3-none-any.whl", hash = "sha256:742890cfd4450d12f58d42da7cfa474fe1ee5d6442e016bf70ab76e5c876c0ea", size = 8896, upload-time = "2026-03-12T17:28:02.328Z" }, ] +[[package]] +name = "more-itertools" +version = "11.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/f7/139d22fef48ac78127d18e01d80cf1be40236ae489769d17f35c3d425293/more_itertools-11.0.2.tar.gz", hash = "sha256:392a9e1e362cbc106a2457d37cabf9b36e5e12efd4ebff1654630e76597df804", size = 144659, upload-time = "2026-04-09T15:01:33.297Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl", hash = "sha256:6e35b35f818b01f691643c6c611bc0902f2e92b46c18fffa77ae1e7c46e912e4", size = 71939, upload-time = "2026-04-09T15:01:32.21Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -2376,6 +2520,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, ] +[[package]] +name = "msgpack" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/31/b46518ecc604d7edf3a4f94cb3bf021fc62aa301f0cb849936968164ef23/msgpack-1.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4efd7b5979ccb539c221a4c4e16aac1a533efc97f3b759bb5a5ac9f6d10383bf", size = 81212, upload-time = "2025-10-08T09:15:14.552Z" }, + { url = "https://files.pythonhosted.org/packages/92/dc/c385f38f2c2433333345a82926c6bfa5ecfff3ef787201614317b58dd8be/msgpack-1.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42eefe2c3e2af97ed470eec850facbe1b5ad1d6eacdbadc42ec98e7dcf68b4b7", size = 84315, upload-time = "2025-10-08T09:15:15.543Z" }, + { url = "https://files.pythonhosted.org/packages/d3/68/93180dce57f684a61a88a45ed13047558ded2be46f03acb8dec6d7c513af/msgpack-1.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fdf7d83102bf09e7ce3357de96c59b627395352a4024f6e2458501f158bf999", size = 412721, upload-time = "2025-10-08T09:15:16.567Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ba/459f18c16f2b3fc1a1ca871f72f07d70c07bf768ad0a507a698b8052ac58/msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fac4be746328f90caa3cd4bc67e6fe36ca2bf61d5c6eb6d895b6527e3f05071e", size = 424657, upload-time = "2025-10-08T09:15:17.825Z" }, + { url = "https://files.pythonhosted.org/packages/38/f8/4398c46863b093252fe67368b44edc6c13b17f4e6b0e4929dbf0bdb13f23/msgpack-1.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fffee09044073e69f2bad787071aeec727183e7580443dfeb8556cbf1978d162", size = 402668, upload-time = "2025-10-08T09:15:19.003Z" }, + { url = "https://files.pythonhosted.org/packages/28/ce/698c1eff75626e4124b4d78e21cca0b4cc90043afb80a507626ea354ab52/msgpack-1.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5928604de9b032bc17f5099496417f113c45bc6bc21b5c6920caf34b3c428794", size = 419040, upload-time = "2025-10-08T09:15:20.183Z" }, + { url = "https://files.pythonhosted.org/packages/67/32/f3cd1667028424fa7001d82e10ee35386eea1408b93d399b09fb0aa7875f/msgpack-1.1.2-cp313-cp313-win32.whl", hash = "sha256:a7787d353595c7c7e145e2331abf8b7ff1e6673a6b974ded96e6d4ec09f00c8c", size = 65037, upload-time = "2025-10-08T09:15:21.416Z" }, + { url = "https://files.pythonhosted.org/packages/74/07/1ed8277f8653c40ebc65985180b007879f6a836c525b3885dcc6448ae6cb/msgpack-1.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:a465f0dceb8e13a487e54c07d04ae3ba131c7c5b95e2612596eafde1dccf64a9", size = 72631, upload-time = "2025-10-08T09:15:22.431Z" }, + { url = "https://files.pythonhosted.org/packages/e5/db/0314e4e2db56ebcf450f277904ffd84a7988b9e5da8d0d61ab2d057df2b6/msgpack-1.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:e69b39f8c0aa5ec24b57737ebee40be647035158f14ed4b40e6f150077e21a84", size = 64118, upload-time = "2025-10-08T09:15:23.402Z" }, +] + [[package]] name = "msgraph-core" version = "1.3.8" @@ -3018,6 +3179,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl", hash = "sha256:2c549f800b70a5c4952197248825584cb00f033b29c692671d3bf08bf380baff", size = 106894, upload-time = "2026-02-09T15:45:21.391Z" }, ] +[[package]] +name = "partd" +version = "1.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "locket" }, + { name = "toolz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/3a/3f06f34820a31257ddcabdfafc2672c5816be79c7e353b02c1f318daa7d4/partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c", size = 21029, upload-time = "2024-05-06T19:51:41.945Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/e7/40fb618334dcdf7c5a316c0e7343c5cd82d3d866edc100d98e29bc945ecd/partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f", size = 18905, upload-time = "2024-05-06T19:51:39.271Z" }, +] + [[package]] name = "pathspec" version = "1.0.4" @@ -3483,6 +3657,17 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pynisher" +version = "1.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "psutil" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/1d5ed36041e11c943833a93137ff2562db97da48851d68431f1feaa8f82f/pynisher-1.0.10.tar.gz", hash = "sha256:2722b795d22129ddd5247038bba727ae06ecdb3a5942070817be7c8d4a680e31", size = 30841, upload-time = "2023-11-13T17:58:47.159Z" } + [[package]] name = "pyobjc" version = "12.1" @@ -6461,6 +6646,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smac" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "configspace" }, + { name = "dask", extra = ["distributed"] }, + { name = "dask-jobqueue" }, + { name = "emcee" }, + { name = "joblib" }, + { name = "numpy" }, + { name = "psutil" }, + { name = "pynisher" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "scikit-learn" }, + { name = "scipy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/3f/1154b649c899f134006b8fb1e79e1a7cccafde7e2ee6b6064023773c4dab/smac-2.4.0-py3-none-any.whl", hash = "sha256:670fb21b812f2130fc5b07d31a9da51821685c5edcccacc9720a199291273b2c", size = 278108, upload-time = "2026-04-22T19:25:23.114Z" }, +] + [[package]] name = "smart-open" version = "7.5.1" @@ -6491,6 +6698,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/c3/6eeb6034408dac0fa653d126c9204ade96b819c936e136c5e8a6897eee9c/socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3", size = 12763, upload-time = "2020-04-17T15:50:31.878Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "soupsieve" version = "2.7" @@ -6642,6 +6858,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "tblib" +version = "3.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8a/14c15ae154895cc131174f858c707790d416c444fc69f93918adfd8c4c0b/tblib-3.2.2.tar.gz", hash = "sha256:e9a652692d91bf4f743d4a15bc174c0b76afc750fe8c7b6d195cc1c1d6d2ccec", size = 35046, upload-time = "2025-11-12T12:21:16.572Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/be/5d2d47b1fb58943194fb59dcf222f7c4e35122ec0ffe8c36e18b5d728f0b/tblib-3.2.2-py3-none-any.whl", hash = "sha256:26bdccf339bcce6a88b2b5432c988b266ebbe63a4e593f6b578b1d2e723d2b76", size = 12893, upload-time = "2025-11-12T12:21:14.407Z" }, +] + [[package]] name = "tenacity" version = "9.1.2" @@ -6757,6 +6982,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/f2/fd673d979185f5dcbac4be7d09461cbb99751554ffb6718d0013af8604cb/tokenizers-0.21.4-cp39-abi3-win_amd64.whl", hash = "sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597", size = 2507568, upload-time = "2025-07-28T15:48:55.456Z" }, ] +[[package]] +name = "toolz" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/d6/114b492226588d6ff54579d95847662fc69196bdeec318eb45393b24c192/toolz-1.1.0.tar.gz", hash = "sha256:27a5c770d068c110d9ed9323f24f1543e83b2f300a687b7891c1a6d56b697b5b", size = 52613, upload-time = "2025-10-17T04:03:21.661Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/12/5911ae3eeec47800503a238d971e51722ccea5feb8569b735184d5fcdbc0/toolz-1.1.0-py3-none-any.whl", hash = "sha256:15ccc861ac51c53696de0a5d6d4607f99c210739caf987b5d2054f3efed429d8", size = 58093, upload-time = "2025-10-17T04:03:20.435Z" }, +] + [[package]] name = "torch" version = "2.8.0" @@ -7372,6 +7606,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, ] +[[package]] +name = "zict" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/ac/3c494dd7ec5122cff8252c1a209b282c0867af029f805ae9befd73ae37eb/zict-3.0.0.tar.gz", hash = "sha256:e321e263b6a97aafc0790c3cfb3c04656b7066e6738c37fffcca95d803c9fba5", size = 33238, upload-time = "2023-04-17T21:41:16.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/ab/11a76c1e2126084fde2639514f24e6111b789b0bfa4fc6264a8975c7e1f1/zict-3.0.0-py2.py3-none-any.whl", hash = "sha256:5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae", size = 43332, upload-time = "2023-04-17T21:41:13.444Z" }, +] + [[package]] name = "zipp" version = "3.23.0" From 1f02723c2cd7845e69a9ebe696482a0b44aa824a Mon Sep 17 00:00:00 2001 From: farshad68 Date: Wed, 6 May 2026 10:41:58 +0200 Subject: [PATCH 08/47] refactor code --- .../optimization/baseline_optimization.py | 92 ++++++------------- 1 file changed, 27 insertions(+), 65 deletions(-) diff --git a/learn2rag/optimization/baseline_optimization.py b/learn2rag/optimization/baseline_optimization.py index ea5b0c4..337a81e 100644 --- a/learn2rag/optimization/baseline_optimization.py +++ b/learn2rag/optimization/baseline_optimization.py @@ -1,5 +1,5 @@ """ -RAG Pipeline Optimization with BERTScore evaluation. +RAG Pipeline Optimization with BERTScore evaluation. """ import argparse @@ -8,6 +8,7 @@ import pathlib import time import copy +import os import numpy as np from bert_score import score as bert_score @@ -19,60 +20,19 @@ import learn2rag.pipeline.search import learn2rag.pipeline.generate +def load_registry(registry_path: pathlib.Path): + if not registry_path.exists(): + logging.warning("no registry found !") + return {"datasets": {}, "prompts": {"default": "Context: {context}\nQ: {question}"}} + return json.loads(registry_path.read_text()) -DATASET_CONFIG = { - "WikiEval": { - "subdirectory": "", - "split": "train", - "question_field": "question", - "answer_field": "answer", - "id_field": "id", - }, - "rag-mini-bioasq": { - "subdirectory": "question-answer-passages", - "split": "test", - "question_field": "question", - "answer_field": "answer", - "id_field": "id", - }, - "hotpot_qa": { # Not being used - "subdirectory": "distractor", - "split": "validation", - "question_field": "question", - "answer_field": "answer", - "id_field": "id", - }, - "repliqa": { # Not being used - "subdirectory": "repliqa_4", - "split": None, - "question_field": "question", - "answer_field": "long_answer", - "id_field": "question_id", - }, -} - -PROMPT_MAP = { - "default": ( - "# Role and Objective\nYou will act as a smart AI chatbot that answers " - "questions only by using the content from the provided information list.\n\n" - "# Instructions\n- Respond in the language of the question.\n" - "- Answer clear and concise.\n- Only use the provided information.\n" - "- NEVER use your general knowledge.\n\n" - "# Information:\n{context}" - ), - "concise": ( - "Answer the question using ONLY the provided information. " - "Be concise and direct. If the information does not contain the answer, say so.\n\n" - "Information:\n{context}" - ), - "detailed": ( - "You are a knowledgeable assistant. Using ONLY the provided information below, " - "answer the question thoroughly. Cite your sources. " - "If the information is insufficient, state that clearly.\n\n" - "Information:\n{context}" - ), -} - +def get_base_user_config(): + """Loads the base config provided by the orchestrator env var.""" + path = os.environ.get("PIPELINE_USER_CONFIG") + if path and pathlib.Path(path).exists(): + return json.loads(pathlib.Path(path).read_text()) + logging.warning("no user config found !") + return {} def run_pipeline(question, user_config, working_config): t0 = time.time() @@ -92,20 +52,21 @@ def run_pipeline(question, user_config, working_config): ]) return answer, context[:3000], search_time, gen_time - -def objective(config, seed, questions, dataset_name, state, answers_dir): +#I removed seed because there are no use for it +# removed dataset_name becuase it just use in yser config and now we inject it +def objective(config, questions, state, answers_dir, prompts_repo, base_user_cfg): state["trial_count"] += 1 tid = state["trial_count"] cfg = dict(config) logging.info(f"Trial {tid}: {cfg}") - wcfg = copy.deepcopy(opt_config) - wcfg["top_k"] = cfg["top_k"] - wcfg["chunk_size"] = cfg["chunk_size"] - wcfg["chunk_overlap"] = cfg["chunk_overlap"] - wcfg["prompt"] = PROMPT_MAP[cfg["prompt_template"]] - ucfg = {"file_path": None, "collection_name": dataset_name, - "imported_documents_file_path": None, "llm": None} + working_cfg = copy.deepcopy(opt_config) + working_cfg.update({ + "top_k": cfg["top_k"], + "chunk_size": cfg["chunk_size"], + "chunk_overlap": cfg["chunk_overlap"], + "prompt": prompts_repo.get(cfg["prompt_template"], prompts_repo["default"]) + }) predictions, goldens = [], [] qa_pairs = [] @@ -116,7 +77,7 @@ def objective(config, seed, questions, dataset_name, state, answers_dir): if not q["question"]: continue try: - answer, context, st, gt = run_pipeline(q["question"], ucfg, wcfg) + answer, context, st, gt = run_pipeline(q["question"], base_user_cfg, working_cfg) t_search += st t_gen += gt predictions.append(answer) @@ -130,6 +91,7 @@ def objective(config, seed, questions, dataset_name, state, answers_dir): }) except Exception as e: logging.warning(f"Trial {tid}, q{idx} failed: {e}") + # do we need to add this :/ predictions.append("") goldens.append(q["ground_truth"]) qa_pairs.append({ @@ -207,7 +169,7 @@ def param_importance(smac, output_path): return result -def run(dataset_name, max_questions, n_trials, output_dir): +def run(dataset_name: str, max_questions: int, n_trials, output_dir: int): if dataset_name not in DATASET_CONFIG: raise ValueError( f"Unknown dataset: {dataset_name}. " From c40629c6068c2cb9d8a3708ac1a9c93126bdf594 Mon Sep 17 00:00:00 2001 From: farshad68 Date: Thu, 7 May 2026 15:09:35 +0200 Subject: [PATCH 09/47] =?UTF-8?q?The=20original=20script=20worked,=20but?= =?UTF-8?q?=20it=20was=20hard=E2=80=91coded=20(datasets,=20prompts,=20user?= =?UTF-8?q?=E2=80=91config=20keys,=20result=20JSON=20schema)=20and=20there?= =?UTF-8?q?fore=20difficult=20to=20extend=20or=20reuse=20in=20different=20?= =?UTF-8?q?environments.=20The=20new=20version=20aimed=20to:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Load configuration dynamically from a registry.json file. --- learn2rag/optimization/__init__.py | 3 +- .../optimization/baseline_optimization.py | 201 +++++++++--------- 2 files changed, 105 insertions(+), 99 deletions(-) diff --git a/learn2rag/optimization/__init__.py b/learn2rag/optimization/__init__.py index 84a6ef1..86410e8 100644 --- a/learn2rag/optimization/__init__.py +++ b/learn2rag/optimization/__init__.py @@ -14,6 +14,7 @@ def main() -> None: parser = argparse.ArgumentParser(description="Run RAG baseline optimization.") parser.add_argument("task", help="Module task name (e.g., learn2rag.optimization)") parser.add_argument("--logging-config", type=str, help="Path to logging config yml") + parser.add_argument("--registry-path", type=str, help="Path to registry.json") parser.add_argument("--dataset", type=str, default="WikiEval") parser.add_argument("--questions", type=int, default=10) parser.add_argument("--trials", type=int, default=10) @@ -34,7 +35,7 @@ def main() -> None: output_dir = pathlib.Path("./optimization/output/") output_dir.mkdir(parents=True, exist_ok=True) - baseline_optimization.run(args.dataset,args.questions,args.trials,output_dir) + baseline_optimization.run(args.dataset,args.questions,args.trials,output_dir,args.registry_path) logging.info("optimization is done") results_path = output_dir /args.dataset/ "optimization_results.json" logging.info(f"save optimized results here : {results_path}") diff --git a/learn2rag/optimization/baseline_optimization.py b/learn2rag/optimization/baseline_optimization.py index 337a81e..be02a12 100644 --- a/learn2rag/optimization/baseline_optimization.py +++ b/learn2rag/optimization/baseline_optimization.py @@ -9,10 +9,11 @@ import time import copy import os +from typing import Dict, Any, List, Union, Tuple import numpy as np from bert_score import score as bert_score -from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation +from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation, Configuration from smac import HyperparameterOptimizationFacade, Scenario from learn2rag.evaluation.tools import read_dataset_qa @@ -20,21 +21,14 @@ import learn2rag.pipeline.search import learn2rag.pipeline.generate -def load_registry(registry_path: pathlib.Path): - if not registry_path.exists(): - logging.warning("no registry found !") - return {"datasets": {}, "prompts": {"default": "Context: {context}\nQ: {question}"}} - return json.loads(registry_path.read_text()) - -def get_base_user_config(): - """Loads the base config provided by the orchestrator env var.""" - path = os.environ.get("PIPELINE_USER_CONFIG") - if path and pathlib.Path(path).exists(): - return json.loads(pathlib.Path(path).read_text()) - logging.warning("no user config found !") - return {} - -def run_pipeline(question, user_config, working_config): +def load_registry(path: str = "registry.json") -> dict: + p = pathlib.Path(path) + if not p.is_file(): + logging.error("registry file not found") + with p.open() as f: + return json.load(f) + +def run_pipeline(question: str, user_config: Dict[str, Any], working_config: Dict[str, Any]) -> Tuple[str, str, float, float]: t0 = time.time() docs = learn2rag.pipeline.search.search(question, user_config, working_config) search_time = time.time() - t0 @@ -54,7 +48,13 @@ def run_pipeline(question, user_config, working_config): #I removed seed because there are no use for it # removed dataset_name becuase it just use in yser config and now we inject it -def objective(config, questions, state, answers_dir, prompts_repo, base_user_cfg): +def objective(config: Configuration, + questions: List[Dict[str, Any]], + dataset_name: str, + state: Dict[str, Any], + answers_dir: pathlib.Path + ,prompt_map +) -> float: state["trial_count"] += 1 tid = state["trial_count"] cfg = dict(config) @@ -65,42 +65,41 @@ def objective(config, questions, state, answers_dir, prompts_repo, base_user_cfg "top_k": cfg["top_k"], "chunk_size": cfg["chunk_size"], "chunk_overlap": cfg["chunk_overlap"], - "prompt": prompts_repo.get(cfg["prompt_template"], prompts_repo["default"]) + "prompt": prompt_map[cfg["prompt_template"]], }) + ucfg = { + "file_path": None, + "collection_name": dataset_name, + "imported_documents_file_path": None, + "llm": None, + } + env_user_cfg = os.environ.get("PIPELINE_USER_CONFIG") + if env_user_cfg and pathlib.Path(env_user_cfg).exists(): + ucfg.update(json.loads(pathlib.Path(env_user_cfg).read_text())) + predictions, goldens = [], [] qa_pairs = [] t_start = time.time() t_search, t_gen = 0.0, 0.0 - for idx, q in enumerate(questions): - if not q["question"]: + for q in questions: + # Preserve the original “skip empty question” guard + if not q.get("question"): continue try: - answer, context, st, gt = run_pipeline(q["question"], base_user_cfg, working_cfg) - t_search += st - t_gen += gt + answer, context, t_s, t_g = run_pipeline(q["question"], ucfg, working_cfg) + t_search += t_s + t_gen += t_g predictions.append(answer) goldens.append(q["ground_truth"]) - qa_pairs.append({ - "id": q["id"], - "question": q["question"], - "golden_answer": q["ground_truth"], - "generated_answer": answer, - "retrieved_context": context, - }) + qa_pairs.append({**q, "generated_answer": answer, "retrieved_context": context}) except Exception as e: - logging.warning(f"Trial {tid}, q{idx} failed: {e}") - # do we need to add this :/ + # Same behaviour as the old version: record a blank answer. TODO : check if we need this + logging.warning(f"Trial {tid}, q{q.get('id','?')} failed: {e}") predictions.append("") goldens.append(q["ground_truth"]) - qa_pairs.append({ - "id": q["id"], - "question": q["question"], - "golden_answer": q["ground_truth"], - "generated_answer": "", - "retrieved_context": "", - }) + qa_pairs.append({**q, "generated_answer": "", "retrieved_context": ""}) if not predictions: return 1.0 @@ -143,7 +142,7 @@ def objective(config, questions, state, answers_dir, prompts_repo, base_user_cfg return float(cost) -def param_importance(smac, output_path): +def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathlib.Path) -> Dict[str, Any]: params = list(smac.scenario.configspace.keys()) configs, costs = [], [] for key, val in smac.runhistory.items(): @@ -169,17 +168,17 @@ def param_importance(smac, output_path): return result -def run(dataset_name: str, max_questions: int, n_trials, output_dir: int): - if dataset_name not in DATASET_CONFIG: - raise ValueError( - f"Unknown dataset: {dataset_name}. " - f"Available: {list(DATASET_CONFIG.keys())}" - ) +def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[str, pathlib.Path],registry_path:str) -> Tuple[ + Dict[str, Any], List[Any], Dict[str, Any]]: + registry = load_registry(registry_path) + datasets = registry["datasets"] + if dataset_name not in datasets: + raise ValueError(f"Unknown dataset: {dataset_name}. Available: {list(datasets.keys())}") + dcfg = datasets[dataset_name] + fields = dcfg["fields"] - dcfg = DATASET_CONFIG[dataset_name] out = pathlib.Path(output_dir) / dataset_name out.mkdir(parents=True, exist_ok=True) - answers_dir = out / "trial_answers" answers_dir.mkdir(parents=True, exist_ok=True) @@ -187,73 +186,79 @@ def run(dataset_name: str, max_questions: int, n_trials, output_dir: int): if max_questions: qa = qa.select(range(min(max_questions, len(qa)))) - questions = [] - for i, r in enumerate(qa): - questions.append({ - "question": r.get(dcfg["question_field"], ""), - "ground_truth": r.get(dcfg["answer_field"], ""), - "id": r.get(dcfg["id_field"], str(i)), - }) - logging.info(f"Loaded {len(questions)} questions from {dataset_name}") + questions = [ + { + "question": r[fields["q"]], + "ground_truth": r[fields["a"]], + "id": r.get(fields["id"], str(i)), + } + for i, r in enumerate(qa) + ] + prompt_map = registry["prompts"] cs = ConfigurationSpace(seed=42) - cs.add([Integer("top_k", (1, 20), default=4), - Integer("chunk_size", (200, 4000), default=2000), - Integer("chunk_overlap", (0, 500), default=200), - Categorical("prompt_template", ["default", "concise", "detailed"], default="default")]) + cs.add([ + Integer("top_k", (1, 20), default=4), + Integer("chunk_size", (200, 4000), default=2000), + Integer("chunk_overlap", (0, 500), default=200), + Categorical("prompt_template", list(prompt_map.keys()), default="default"), + ]) cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) + scenario = Scenario( + cs, + deterministic=True, + n_trials=n_trials, + walltime_limit=7200, + seed=42, + output_directory=out / "smac_output", + ) + state: Dict[str, Any] = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} - scenario = Scenario(configspace=cs, deterministic=True, n_trials=n_trials, - walltime_limit=7200, seed=42, output_directory=out / "smac_output") - - state = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} smac = HyperparameterOptimizationFacade( scenario=scenario, - target_function=lambda config, seed=0: objective( - config, seed, questions, dataset_name, state, answers_dir - ), + target_function=lambda config, seed=0: objective(config, questions, dataset_name, state, answers_dir,prompt_map) ) - t0 = time.time() incumbent = smac.optimize() - total_time = time.time() - t0 - importance = param_importance(smac, out) - - with open(out / "optimization_results.json", "w") as f: - json.dump({ - "best_config": dict(incumbent), - "run_history": state["history"], - "convergence": state["convergence"], - "parameter_importance": importance, - "total_time_s": round(total_time, 2), - "dataset": dataset_name, - "metric": "bertscore_golden", - "answers_dir": str(answers_dir), - }, f, indent=2, default=str) - - logging.info(f"Done in {total_time:.0f}s") - logging.info(f"Trial answers saved to {answers_dir}") - return incumbent, state["history"], importance - + total_time = time.time() - t0 + best_cfg = incumbent.get_dictionary() + results_path = out / "optimization_results.json" + results_path.write_text(json.dumps({ + "best_config": best_cfg, + "run_history": state["history"], + "convergence": state["convergence"], + "parameter_importance": importance, + "total_time_s": round(total_time, 2), + "dataset": dataset_name, + "metric": "bertscore_golden", + "answers_dir":str(answers_dir), + }, indent=2, default = str)) + + return best_cfg, state["history"], importance if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--dataset", type=str, default="WikiEval", - choices=list(DATASET_CONFIG.keys())) + parser.add_argument("task", nargs='?', default="learn2rag.optimization") + parser.add_argument("--dataset", type=str, default="WikiEval") parser.add_argument("--max_questions", type=int, default=50) parser.add_argument("--n_trials", type=int, default=10) + parser.add_argument("--logging-config", type=str) + parser.add_argument("--registry", type=str, default="registry.json") parser.add_argument("--output_dir", type=str, default="optimization_results_baseline") - args = parser.parse_args() + args, _ = parser.parse_known_args() - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s", - force=True, - ) - incumbent, history, importance = run( - args.dataset, args.max_questions, args.n_trials, args.output_dir, - ) + final_output_dir = pathlib.Path(args.output_dir) + + env_out = os.environ.get("PIPELINE_OPT_CONFIG") + if not final_output_dir.exists() and env_out: + final_output_dir = pathlib.Path(env_out).parent + + incumbent, history, importance = run(args.dataset, args.max_questions, args.n_trials, final_output_dir, args.registry) + + # incumbent, history, importance = run( + # args.dataset, args.max_questions, args.n_trials, args.output_dir, + # ) best = min(history, key=lambda x: x["cost"]) print(f"\nBest config: {dict(incumbent)}") From c9568cf00f3694f35d354b87ada00785fcaf3ae9 Mon Sep 17 00:00:00 2001 From: Carolin Walter Date: Tue, 26 May 2026 14:17:20 +0000 Subject: [PATCH 10/47] improve ingestion performance --- learn2rag/pipeline/embeddings.py | 51 +++++++--- learn2rag/pipeline/ingestion.py | 157 ++++++++++++++++------------- learn2rag/pipeline/opt_config.json | 1 + 3 files changed, 123 insertions(+), 86 deletions(-) diff --git a/learn2rag/pipeline/embeddings.py b/learn2rag/pipeline/embeddings.py index 75423ae..f45e87f 100644 --- a/learn2rag/pipeline/embeddings.py +++ b/learn2rag/pipeline/embeddings.py @@ -1,12 +1,29 @@ +from functools import lru_cache from FlagEmbedding import BGEM3FlagModel # type: ignore[import-untyped] from sentence_transformers import SentenceTransformer -from typing import List, Union, Any, cast, Literal +from typing import List, Any, Literal import numpy as np import warnings -def create_embeddings(input_sample: List[str], model_name: str = "BAAI/bge-m3", embedding_mode: str = "dense") -> dict[Literal['dense_vecs', 'lexical_weights', 'colbert_vecs'], np.ndarray[Any, Any] | list[dict[str, float]] | list[np.ndarray[Any, Any]]]: + +@lru_cache(maxsize=4) +def _get_bge_m3_model() -> BGEM3FlagModel: + return BGEM3FlagModel("BAAI/bge-m3", use_fp16=True) + + +@lru_cache(maxsize=4) +def _get_sentence_transformer_model(model_name: str) -> SentenceTransformer: + return SentenceTransformer(model_name) + + +def create_embeddings( + input_sample: List[str], + model_name: str = "BAAI/bge-m3", + embedding_mode: str = "dense", +) -> dict[Literal['dense_vecs', 'lexical_weights', 'colbert_vecs'], np.ndarray[Any, Any] | list[dict[str, float]] | list[np.ndarray[Any, Any]]] | np.ndarray[Any, Any]: if model_name == "BAAI/bge-m3": - model = BGEM3FlagModel("BAAI/bge-m3", use_fp16=True) + model = _get_bge_m3_model() + if embedding_mode == "dense": return model.encode( # type: ignore[no-any-return] input_sample, @@ -16,7 +33,7 @@ def create_embeddings(input_sample: List[str], model_name: str = "BAAI/bge-m3", return_colbert_vecs=False, ) elif embedding_mode == "sparse": - return model.encode( # type: ignore[no-any-return] + return model.encode( # type: ignore[no-any-return] input_sample, batch_size=512, return_dense=False, @@ -31,25 +48,25 @@ def create_embeddings(input_sample: List[str], model_name: str = "BAAI/bge-m3", return_sparse=True, return_colbert_vecs=False, ) - elif embedding_mode == "dense_sparse_colbert": + elif embedding_mode == "colbert": return model.encode( # type: ignore[no-any-return] input_sample, - batch_size=512, - return_dense=True, - return_sparse=True, + batch_size=32, + return_dense=False, + return_sparse=False, return_colbert_vecs=True, ) - elif embedding_mode == "colbert": + elif embedding_mode == "dense_sparse_colbert": return model.encode( # type: ignore[no-any-return] input_sample, - batch_size=512, - return_dense=False, - return_sparse=False, + batch_size=8, + return_dense=True, + return_sparse=True, return_colbert_vecs=True, ) else: warnings.warn( - f"Embedding mode unknown or not provided. Using dense embeddings" + "Embedding mode unknown or not provided. Using dense embeddings" ) return model.encode( # type: ignore[no-any-return] input_sample, @@ -58,12 +75,14 @@ def create_embeddings(input_sample: List[str], model_name: str = "BAAI/bge-m3", return_sparse=False, return_colbert_vecs=False, ) + elif model_name == "sentence-transformers/all-mpnet-base-v2": - model = SentenceTransformer("sentence-transformers/all-mpnet-base-v2") + model = _get_sentence_transformer_model("sentence-transformers/all-mpnet-base-v2") return model.encode(input_sample) # type: ignore[return-value] + else: - warnings.warn(f"Embedding model unknown or not provided. Using dense embeddings of default model: BAAI/bge-m3") - model = BGEM3FlagModel("BAAI/bge-m3", use_fp16=True) + warnings.warn("Embedding model unknown or not provided. Using dense embeddings of default model: BAAI/bge-m3") + model = _get_bge_m3_model() return model.encode( # type: ignore[no-any-return] input_sample, batch_size=512, diff --git a/learn2rag/pipeline/ingestion.py b/learn2rag/pipeline/ingestion.py index d4f5cfa..9690a28 100644 --- a/learn2rag/pipeline/ingestion.py +++ b/learn2rag/pipeline/ingestion.py @@ -8,7 +8,7 @@ from collections.abc import Iterator from langchain.text_splitter import RecursiveCharacterTextSplitter -from langchain_core.documents.base import Document +from langchain_core.documents import Document from .qdrant import Qdrant from qdrant_client.models import PointStruct, Filter, FieldCondition, MatchValue, SparseVector, VectorParams, MultiVectorConfig, MultiVectorComparator, Distance @@ -128,85 +128,102 @@ def ingest_batch(docs: list[Document], qdrant: Qdrant, user_config: dict[str, An ``embedding_model``, and ``search_mode``). """ collection_name = user_config["collection_name"] - all_documents = docs logging.info('Splitting documents into chunks') text_splitter = RecursiveCharacterTextSplitter( chunk_size=opt_config["chunk_size"], chunk_overlap=opt_config["chunk_overlap"] ) - chunks = text_splitter.split_documents(all_documents) - - chunks_content = [chunk.page_content for chunk in chunks] - if len(opt_config["multi_search"]) > 0 and opt_config["query_mode"] == "multi": - chunks_metadata: dict[str, list[str]] = {} - embeddings_metadata: dict[str, Any] = {} - for item in opt_config["multi_search"]: - chunks_metadata[item] = list(get_chunks_metadata(chunks, item)) - embeddings_metadata[item] = create_embeddings(chunks_metadata[item], opt_config["embedding_model"], opt_config["search_mode"]) - dense_vecs = embeddings_metadata[item]["dense_vecs"] - if isinstance(dense_vecs, np.ndarray): - assert dense_vecs.ndim == 2, dense_vecs.shape + chunks = text_splitter.split_documents(docs) + + ingestion_batch_size = opt_config["ingestion_batch_size"] + logging.info('Creating embeddings and ingesting in batches...') + for batch_start in range(0, len(chunks), ingestion_batch_size): + batch_chunks = chunks[batch_start:batch_start + ingestion_batch_size] + batch_content = [chunk.page_content for chunk in batch_chunks] + batch_chunk_hash = [hashlib.md5(chunk.page_content.encode()).hexdigest() for chunk in batch_chunks] + + embeddings = create_embeddings( + batch_content, + opt_config["embedding_model"], + opt_config["search_mode"], + ) + + if len(opt_config["multi_search"]) > 0 and opt_config["query_mode"] == "multi": + if not isinstance(embeddings, dict) or "dense_vecs" not in embeddings: + raise TypeError("Expected dense_vecs in embeddings for multi query mode") + + embeddings_metadata: dict[str, Any] = {} + for item in opt_config["multi_search"]: + item_values = list(get_chunks_metadata(batch_chunks, item)) + embeddings_metadata[item] = create_embeddings( + item_values, + opt_config["embedding_model"], + opt_config["search_mode"], + ) + dense_vecs = embeddings_metadata[item]["dense_vecs"] + if isinstance(dense_vecs, np.ndarray): + assert dense_vecs.ndim == 2, dense_vecs.shape + else: + raise TypeError(f"dense_vecs must be np.ndarray, got {type(dense_vecs)}") + + mmembeddings: list[np.ndarray[Any, Any]] = [] + for i in range(len(embeddings['dense_vecs'])): + vecs_to_concat: list[np.ndarray[Any, Any]] = [cast(np.ndarray[Any, Any], embeddings['dense_vecs'][i])] + for item in embeddings_metadata.keys(): + vecs_to_concat.append(cast(np.ndarray[Any, Any], embeddings_metadata[item]['dense_vecs'][i])) + mmembeddings.append(np.concatenate(vecs_to_concat, axis=0)) + embeddings['dense_vecs'] = mmembeddings + + if isinstance(embeddings, dict) and "dense_vecs" in embeddings: + if opt_config["search_mode"] == "dense": + chunks_with_embeddings = [ + dict(chunk) | {"dense_vec": dense, "chunk_hash": c_hash} + for chunk, dense, c_hash in zip(batch_chunks, embeddings["dense_vecs"], batch_chunk_hash) + ] + elif opt_config["search_mode"] == "dense_sparse": + chunks_with_embeddings = [ + dict(chunk) + | {"dense_vec": dense, "lexical_weights": sparse, "chunk_hash": c_hash} + for chunk, dense, sparse, c_hash in zip( + batch_chunks, + list(embeddings["dense_vecs"]), + list(embeddings["lexical_weights"]), + batch_chunk_hash + ) + ] + elif opt_config["search_mode"] == "dense_sparse_colbert": + chunks_with_embeddings = [ + dict(chunk) + | {"dense_vec": dense, "lexical_weights": sparse, "colbert_vecs": colbert, "chunk_hash": c_hash} + for chunk, dense, sparse, colbert, c_hash in zip( + batch_chunks, + list(embeddings["dense_vecs"]), + list(embeddings["lexical_weights"]), + list(embeddings["colbert_vecs"]), + batch_chunk_hash + ) + ] else: - raise TypeError(f"dense_vecs must be np.ndarray, got {type(dense_vecs)}") - - chunk_hash = [hashlib.md5(chunk.page_content.encode()).hexdigest() for chunk in chunks] - - logging.info('Creating embeddings...') - embeddings = create_embeddings(chunks_content, opt_config["embedding_model"], opt_config["search_mode"]) - if len(opt_config["multi_search"]) > 0 and opt_config["query_mode"] == "multi": - mmembeddings: list[np.ndarray[Any, Any]] = [] - for i in range(len(embeddings['dense_vecs'])): - vecs_to_concat: list[np.ndarray[Any, Any]] = [cast(np.ndarray[Any, Any], embeddings['dense_vecs'][i])] - for item in embeddings_metadata.keys(): - vecs_to_concat.append(cast(np.ndarray[Any, Any], embeddings_metadata[item]['dense_vecs'][i])) - mmembeddings.append(np.concatenate(vecs_to_concat, axis=0)) - embeddings['dense_vecs'] = mmembeddings - - if isinstance(embeddings, dict) and "dense_vecs" in embeddings: - if opt_config["search_mode"] == "dense": + chunks_with_embeddings = [ + dict(chunk) | {"dense_vec": dense, "chunk_hash": c_hash} + for chunk, dense, c_hash in zip(batch_chunks, embeddings["dense_vecs"], batch_chunk_hash) + ] + else: chunks_with_embeddings = [ dict(chunk) | {"dense_vec": dense, "chunk_hash": c_hash} - for chunk, dense, c_hash in zip(chunks, embeddings["dense_vecs"], chunk_hash) - ] - if opt_config["search_mode"] == "dense_sparse": - chunks_with_embeddings = [ - dict(chunk) - | {"dense_vec": dense, "lexical_weights": sparse, "chunk_hash": c_hash} - for chunk, dense, sparse, c_hash in zip( - chunks, - list(embeddings["dense_vecs"]), - list(embeddings["lexical_weights"]), - chunk_hash - ) + for chunk, dense, c_hash in zip(batch_chunks, embeddings, batch_chunk_hash) ] - if opt_config["search_mode"] == "dense_sparse_colbert": - chunks_with_embeddings = [ - dict(chunk) - | {"dense_vec": dense, "lexical_weights": sparse, "colbert_vecs": colbert, "chunk_hash": c_hash} - for chunk, dense, sparse, colbert, c_hash in zip( - chunks, - list(embeddings["dense_vecs"]), - list(embeddings["lexical_weights"]), - list(embeddings['colbert_vecs']), - chunk_hash - ) - ] - else: - chunks_with_embeddings = [ - dict(chunk) | {"dense_vec": dense, "chunk_hash": c_hash} - for chunk, dense, c_hash in zip(chunks, embeddings, chunk_hash) - ] - for sample in chunks_with_embeddings: - if not point_exists(qdrant, collection_name, sample['metadata']['loader_id'], sample['metadata']['source'], sample['metadata']['content_hash'], sample['chunk_hash']): - if opt_config["search_mode"] == "dense_sparse": - insert_dense_sparse(qdrant, collection_name, sample) - elif opt_config["search_mode"] == "dense_sparse_colbert": - insert_dense_sparse_colbert(qdrant, collection_name, sample) - elif opt_config["query_mode"] == "multi": - insert_multi(qdrant, collection_name, sample) - else: - insert(qdrant, collection_name, sample) + for sample in chunks_with_embeddings: + if not point_exists(qdrant, collection_name, sample['metadata']['loader_id'], sample['metadata']['source'], sample['metadata']['content_hash'], sample['chunk_hash']): + if opt_config["search_mode"] == "dense_sparse": + insert_dense_sparse(qdrant, collection_name, sample) + elif opt_config["search_mode"] == "dense_sparse_colbert": + insert_dense_sparse_colbert(qdrant, collection_name, sample) + elif opt_config["query_mode"] == "multi": + insert_multi(qdrant, collection_name, sample) + else: + insert(qdrant, collection_name, sample) def index(documents: list[Document], user_config: dict[str, Any], opt_config: dict[str, Any]) -> None: diff --git a/learn2rag/pipeline/opt_config.json b/learn2rag/pipeline/opt_config.json index f376bdb..2f5738b 100644 --- a/learn2rag/pipeline/opt_config.json +++ b/learn2rag/pipeline/opt_config.json @@ -1,6 +1,7 @@ { "chunk_size": 2000, "chunk_overlap": 200, + "ingestion_batch_size": 32, "embedding_model": "BAAI/bge-m3", "vector_size": { "sentence-transformers/all-mpnet-base-v2": 768, From 14e9f7bc226fadebc74a89b8082745c0a51af1b9 Mon Sep 17 00:00:00 2001 From: Carolin Walter Date: Tue, 26 May 2026 15:08:08 +0000 Subject: [PATCH 11/47] fix mypy error --- learn2rag/pipeline/embeddings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learn2rag/pipeline/embeddings.py b/learn2rag/pipeline/embeddings.py index f45e87f..acf9818 100644 --- a/learn2rag/pipeline/embeddings.py +++ b/learn2rag/pipeline/embeddings.py @@ -78,7 +78,7 @@ def create_embeddings( elif model_name == "sentence-transformers/all-mpnet-base-v2": model = _get_sentence_transformer_model("sentence-transformers/all-mpnet-base-v2") - return model.encode(input_sample) # type: ignore[return-value] + return model.encode(input_sample) else: warnings.warn("Embedding model unknown or not provided. Using dense embeddings of default model: BAAI/bge-m3") From eb21b9f1545abe4d25f5a49b405f9bc8613f3fee Mon Sep 17 00:00:00 2001 From: Hanna Reder Date: Wed, 27 May 2026 07:16:38 +0000 Subject: [PATCH 12/47] WIP: retrieval optimisation --- learn2rag/evaluation/tools.py | 23 +- .../optimization/retrieval_optimization.py | 325 ++++++++++++++++++ 2 files changed, 345 insertions(+), 3 deletions(-) create mode 100644 learn2rag/optimization/retrieval_optimization.py diff --git a/learn2rag/evaluation/tools.py b/learn2rag/evaluation/tools.py index b927bc1..ea1e351 100644 --- a/learn2rag/evaluation/tools.py +++ b/learn2rag/evaluation/tools.py @@ -7,6 +7,7 @@ import datasets import json_stream # type: ignore[import-untyped] from typing import Any, Callable +import pandas as pd import learn2rag.pipeline.ingestion from learn2rag.pipeline.config import opt_config @@ -81,9 +82,25 @@ def ingest_dataset_documents(dataset_name: str) -> None: def read_dataset_qa(dataset_name: str, subdirectory: str, split: str | None=None) -> Any: logging.debug(f'{dataset_name=}') - dataset_work_dir = pathlib.Path('./datasets') / dataset_name - dataset_dict = datasets.load_from_disk(dataset_work_dir / 'source' / subdirectory) - return dataset_dict[split] if split is not None else dataset_dict + + if subdirectory.endswith('.csv'): + df = pd.read_csv(subdirectory, sep=';') + # wrap in object that behaves like HuggingFace dataset + class CSVDataset: + def __init__(self, df): + self.data = df + def __len__(self): + return len(self.data) + def __getitem__(self, idx): + return self.data.iloc[idx].to_dict() + def select(self, indices): + return CSVDataset(self.data.iloc[list(indices)]) + return CSVDataset(df) + + else: + dataset_work_dir = pathlib.Path('./datasets') / dataset_name + dataset_dict = datasets.load_from_disk(dataset_work_dir / 'source' / subdirectory) + return dataset_dict[split] if split is not None else dataset_dict def basic_pipeline(dataset_name: str, question: str) -> dict[str, Any]: diff --git a/learn2rag/optimization/retrieval_optimization.py b/learn2rag/optimization/retrieval_optimization.py new file mode 100644 index 0000000..71502ea --- /dev/null +++ b/learn2rag/optimization/retrieval_optimization.py @@ -0,0 +1,325 @@ +""" +RAG Retrieval Optimization. +""" + +import argparse +import json +import logging +import pathlib +import time +import copy +import os +import asyncio +from typing import Dict, Any, List, Union, Tuple + +import numpy as np +from bert_score import score as bert_score +from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation, Configuration, \ + ForbiddenAndConjunction, ForbiddenEqualsClause +from smac import HyperparameterOptimizationFacade, Scenario + +from learn2rag.evaluation.tools import read_dataset_qa +from learn2rag.pipeline.config import opt_config +import learn2rag.pipeline.search +import learn2rag.pipeline.generate + + + + +def load_registry(path: str = "registry.json") -> dict: + p = pathlib.Path(path) + if not p.is_file(): + logging.error("registry file not found") + with p.open() as f: + return json.load(f) + +def run_search(question: str, user_config: Dict[str, Any], working_config: Dict[str, Any]) -> Tuple[List[Any], float]: + t0 = time.time() + # docs = learn2rag.pipeline.search.search(question, user_config, working_config) + # TODO user config/opt config + docs = asyncio.run(learn2rag.pipeline.search.search_authorized(question, user="anonymous", request_id=None, user_config=user_config, opt_config=working_config)) + search_time = time.time() - t0 + source_list = [point.payload['source'] for point in docs] + return source_list, search_time + + +def recall(search_results, labels): + count = 0 + top_k = opt_config["top_k"] + for q in range(len(search_results)): + label = str(labels[q]) + hits = [str(h) for h in search_results[q]] + print('label ',label, ' hits: ', hits) + if label in hits[:top_k]: + count += 1 + return count / len(labels) if labels else 0.0 + + +#I removed seed because there are no use for it +# removed dataset_name because it just use in user config and now we inject it +def objective(config: Configuration, + questions: List[Dict[str, Any]], + dataset_name: str, + state: Dict[str, Any], + answers_dir: pathlib.Path + ,prompt_map +) -> float: + state["trial_count"] += 1 + tid = state["trial_count"] + cfg = dict(config) + logging.info(f"Trial {tid}: {cfg}") + + if cfg["rewrite"] == "False" and cfg["rewrite_mode"] in {"keywords", "subqueries", "subqueries_keywords"}: + logging.warning(f"Skip invalid cfg: {cfg}") + return 1.0 + + if cfg["reranking"] == "False" and cfg["reranking_mode"] in {"reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"}: + logging.warning(f"Skip invalid cfg: {cfg}") + return 1.0 + + if cfg["search_mode"] in {"dense", "sparse"} and cfg["fusion_mode"] in {"DBSF", "RRF"}: + logging.warning(f"Skip invalid cfg: {cfg}") + return 1.0 + + +# TODO + working_cfg = copy.deepcopy(opt_config) + working_cfg.update({ + "chunk_size": cfg["chunk_size"], + "chunk_overlap": cfg["chunk_overlap"], + "search_mode": cfg["search_mode"], + "reranking_mode": cfg["reranking_mode"], + "rewrite_mode": cfg["rewrite_mode"], + "fusion_mode": cfg["fusion_mode"], + "reranking": cfg["reranking"], + "rewrite": cfg["rewrite"] + }) + +# TODO + ucfg = { + "file_path": None, + "collection_name": dataset_name, # TODO get collection name depending on hyperparameters + "imported_documents_file_path": None, + "llm": None, + } + env_user_cfg = os.environ.get("PIPELINE_USER_CONFIG") + if env_user_cfg and pathlib.Path(env_user_cfg).exists(): + ucfg.update(json.loads(pathlib.Path(env_user_cfg).read_text())) + + predictions, goldens = [], [] + qa_pairs = [] + t_start = time.time() + t_search = 0.0 + + for q in questions: + # Preserve the original “skip empty question” guard + if not q.get("question"): + continue + try: + source_list, t_s = run_search(q["question"], ucfg, working_cfg) + print("SOURCE_LIST: ", source_list) + t_search += t_s + predictions.append(source_list) + goldens.append(q["ground_truth"]) + qa_pairs.append({**q, "retrieved_sources": source_list}) + except Exception as e: + # Same behaviour as the old version: record a blank answer. TODO : check if we need this + logging.warning(f"Trial {tid}, q{q.get('id','?')} failed: {e}") + predictions.append([]) + goldens.append(q["ground_truth"]) + qa_pairs.append({**q, "generated_answer": "", "retrieved_context": ""}) + + if not predictions: + return 1.0 + + t_score = time.time() + recall_score = recall(predictions, goldens) + scoring_time = time.time() - t_score + + # objective function + cost = 1.0 - recall_score + total_time = time.time() - t_start + + trial_answers = { + "trial_id": tid, + "config": cfg, + "cost": float(cost), + "avg_recall": float(recall_score), + "qa_pairs": qa_pairs, + } + answers_file = answers_dir / f"trial_{tid}_answers.json" + with open(answers_file, "w") as f: + json.dump(trial_answers, f, indent=2, default=str) + + state["best_cost"] = min(state["best_cost"], cost) + state["convergence"].append({"trial": tid, "cost": float(cost), "best_cost": float(state["best_cost"])}) + state["history"].append({ + "trial_id": tid, "config": cfg, + "avg_recall": float(recall_score), + "cost": float(cost), "time_s": round(total_time, 2), + "search_s": round(t_search, 2), + "scoring_s": round(scoring_time, 2), + }) + + logging.info( + f"Trial {tid}: recall={recall_score:.4f} cost={cost:.4f} " + f"time={total_time:.1f}s (search={t_search:.1f} score={scoring_time:.1f})" + ) + return float(cost) + + +def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathlib.Path) -> Dict[str, Any]: + params = list(smac.scenario.configspace.keys()) + configs, costs = [], [] + for key, val in smac.runhistory.items(): + configs.append(dict(smac.runhistory.get_config(key.config_id))) + costs.append(val.cost) + if len(configs) < 3: + return {} + + raw = {} + for p in params: + groups = {} + for c, cost in zip(configs, np.array(costs)): + groups.setdefault(str(c[p]), []).append(cost) + means = [np.mean(g) for g in groups.values()] + raw[p] = float(np.var(means)) if len(means) > 1 else 0.0 + + total = sum(raw.values()) + imp = {p: round(v / total, 4) for p, v in raw.items()} if total > 0 else raw + ranking = sorted(imp, key=imp.get, reverse=True) + result = {"method": "variance_based", "ranking": ranking, "individual": imp} + with open(output_path / "parameter_importance.json", "w") as f: + json.dump(result, f, indent=2) + return result + + +def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[str, pathlib.Path], registry_path:str) -> Tuple[ + Dict[str, Any], List[Any], Dict[str, Any]]: + registry = load_registry(registry_path) + datasets = registry["datasets"] + if dataset_name not in datasets: + raise ValueError(f"Unknown dataset: {dataset_name}. Available: {list(datasets.keys())}") + dcfg = datasets[dataset_name] + fields = dcfg["fields"] + + out = pathlib.Path(output_dir) / dataset_name + out.mkdir(parents=True, exist_ok=True) + answers_dir = out / "trial_answers" + answers_dir.mkdir(parents=True, exist_ok=True) + + qa = read_dataset_qa(dataset_name, dcfg["subdirectory"], dcfg["split"]) + if max_questions: + qa = qa.select(range(min(max_questions, len(qa)))) + + questions = [ + { + "question": r[fields["q"]], + "ground_truth": r[fields["a"]], + "id": r.get(fields["id"], str(i)), + } + for i, r in enumerate(qa) + ] + prompt_map = registry["prompts"] + + cs = ConfigurationSpace(seed=42) + cs.add([ + # Categorical("chunk_size", [250, 1000, 2000], default=1000), + # Categorical("chunk_overlap", [50, 200], default=50), + Categorical("chunk_size", [2000], default=2000), + Categorical("chunk_overlap", [50], default=50), + Categorical("search_mode", ["dense", "sparse", "dense_sparse", "dense_sparse_colbert"], default="dense"), + Categorical("reranking_mode", ["none", "reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"], default="reranking_with_sentence_transformers"), + Categorical("rewrite_mode", ["none", "subqueries", "keywords", "subqueries_keywords"], default="subqueries_keywords"), + Categorical("fusion_mode", ["none", "DBSF", "RRF"], default="DBSF"), + Categorical("reranking", ["True", "False"], default="False"), + Categorical("rewrite", ["True", "False"], default="False"), + ]) + cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) + # for sm in ["dense", "sparse"]: + # for fm in ["DBSF", "RRF"]: + # cs.add( + # ForbiddenAndConjunction( + # ForbiddenEqualsClause(cs["search_mode"], sm), + # ForbiddenEqualsClause(cs["fusion_mode"], fm), + # ) + # ) + # for rrm in ["reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"]: + # cs.add( + # ForbiddenAndConjunction( + # ForbiddenEqualsClause(cs["reranking"], "False"), + # ForbiddenEqualsClause(cs["reranking_mode"], rrm), + # ) + # ) + # + # for rwm in ["keywords", "subqueries", "subqueries_keywords"]: + # cs.add( + # ForbiddenAndConjunction( + # ForbiddenEqualsClause(cs["rewrite"], "False"), + # ForbiddenEqualsClause(cs["rewrite_mode"], rwm), + # ) + # ) + + scenario = Scenario( + cs, + deterministic=True, + n_trials=n_trials, + walltime_limit=7200, + seed=42, + output_directory=out / "smac_output", + ) + state: Dict[str, Any] = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + + smac = HyperparameterOptimizationFacade( + scenario=scenario, + target_function=lambda config, seed=0: objective(config, questions, dataset_name, state, answers_dir,prompt_map) + ) + t0 = time.time() + incumbent = smac.optimize() + importance = param_importance(smac, out) + total_time = time.time() - t0 + best_cfg = incumbent.get_dictionary() + results_path = out / "optimization_results.json" + results_path.write_text(json.dumps({ + "best_config": best_cfg, + "run_history": state["history"], + "convergence": state["convergence"], + "parameter_importance": importance, + "total_time_s": round(total_time, 2), + "dataset": dataset_name, + "metric": "recall", + "answers_dir":str(answers_dir), + }, indent=2, default = str)) + + return best_cfg, state["history"], importance + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("task", nargs='?', default="learn2rag.optimization") + parser.add_argument("--dataset", type=str, default="WikiEval") + parser.add_argument("--max_questions", type=int, default=50) + parser.add_argument("--n_trials", type=int, default=10) + parser.add_argument("--logging-config", type=str) + parser.add_argument("--registry", type=str, default="registry.json") + parser.add_argument("--output_dir", type=str, default="optimization_results_baseline") + args, _ = parser.parse_known_args() + + final_output_dir = pathlib.Path(args.output_dir) + + env_out = os.environ.get("PIPELINE_OPT_CONFIG") + if not final_output_dir.exists() and env_out: + final_output_dir = pathlib.Path(env_out).parent + + incumbent, history, importance = run(args.dataset, args.max_questions, args.n_trials, final_output_dir, args.registry) + + # incumbent, history, importance = run( + # args.dataset, args.max_questions, args.n_trials, args.output_dir, + # ) + + best = min(history, key=lambda x: x["cost"]) + print(f"\nBest config: {dict(incumbent)}") + # print(f"BERTScore (golden): {best['avg_bertscore_golden']:.4f}") + if importance: + print(f"\nParameter importance:") + for i, p in enumerate(importance["ranking"], 1): + print(f" {i}. {p}: {importance['individual'][p]:.4f}") \ No newline at end of file From eb844b8edd20892b0daf2920d2740511e8504a53 Mon Sep 17 00:00:00 2001 From: Hanna Reder Date: Wed, 27 May 2026 12:54:30 +0000 Subject: [PATCH 13/47] WIP: improve parameter handling --- .../optimization/retrieval_optimization.py | 69 +++++++++++-------- learn2rag/pipeline/search.py | 2 +- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/learn2rag/optimization/retrieval_optimization.py b/learn2rag/optimization/retrieval_optimization.py index 71502ea..77095b5 100644 --- a/learn2rag/optimization/retrieval_optimization.py +++ b/learn2rag/optimization/retrieval_optimization.py @@ -98,7 +98,7 @@ def objective(config: Configuration, # TODO ucfg = { "file_path": None, - "collection_name": dataset_name, # TODO get collection name depending on hyperparameters + "collection_name": "CSC-CS_2000-CO_50", # TODO get collection name depending on hyperparameters "imported_documents_file_path": None, "llm": None, } @@ -117,7 +117,6 @@ def objective(config: Configuration, continue try: source_list, t_s = run_search(q["question"], ucfg, working_cfg) - print("SOURCE_LIST: ", source_list) t_search += t_s predictions.append(source_list) goldens.append(q["ground_truth"]) @@ -229,36 +228,50 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ Categorical("chunk_size", [2000], default=2000), Categorical("chunk_overlap", [50], default=50), Categorical("search_mode", ["dense", "sparse", "dense_sparse", "dense_sparse_colbert"], default="dense"), - Categorical("reranking_mode", ["none", "reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"], default="reranking_with_sentence_transformers"), - Categorical("rewrite_mode", ["none", "subqueries", "keywords", "subqueries_keywords"], default="subqueries_keywords"), - Categorical("fusion_mode", ["none", "DBSF", "RRF"], default="DBSF"), + Categorical("reranking_mode", ["none", "reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"], default="none"), + Categorical("rewrite_mode", ["none", "subqueries", "keywords", "subqueries_keywords"], default="none"), + Categorical("fusion_mode", ["none", "DBSF", "RRF"], default="none"), Categorical("reranking", ["True", "False"], default="False"), Categorical("rewrite", ["True", "False"], default="False"), ]) cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) - # for sm in ["dense", "sparse"]: - # for fm in ["DBSF", "RRF"]: - # cs.add( - # ForbiddenAndConjunction( - # ForbiddenEqualsClause(cs["search_mode"], sm), - # ForbiddenEqualsClause(cs["fusion_mode"], fm), - # ) - # ) - # for rrm in ["reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"]: - # cs.add( - # ForbiddenAndConjunction( - # ForbiddenEqualsClause(cs["reranking"], "False"), - # ForbiddenEqualsClause(cs["reranking_mode"], rrm), - # ) - # ) - # - # for rwm in ["keywords", "subqueries", "subqueries_keywords"]: - # cs.add( - # ForbiddenAndConjunction( - # ForbiddenEqualsClause(cs["rewrite"], "False"), - # ForbiddenEqualsClause(cs["rewrite_mode"], rwm), - # ) - # ) + + for sm in ["dense", "sparse"]: + for fm in ["DBSF", "RRF"]: + cs.add(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs["search_mode"], sm), + ForbiddenEqualsClause(cs["fusion_mode"], fm), + )) + + for sm in ["dense_sparse", "dense_sparse_colbert"]: + cs.add(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs["search_mode"], sm), + ForbiddenEqualsClause(cs["fusion_mode"], "none"), + )) + + + for rrm in ["reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"]: + cs.add(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs["reranking"], "False"), + ForbiddenEqualsClause(cs["reranking_mode"], rrm), + )) + + cs.add(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs["reranking"], "True"), + ForbiddenEqualsClause(cs["reranking_mode"], "none"), + )) + + for rwm in ["keywords", "subqueries", "subqueries_keywords"]: + cs.add(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs["rewrite"], "False"), + ForbiddenEqualsClause(cs["rewrite_mode"], rwm), + )) + + cs.add(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs["rewrite"], "True"), + ForbiddenEqualsClause(cs["rewrite_mode"], "none"), + )) + scenario = Scenario( cs, diff --git a/learn2rag/pipeline/search.py b/learn2rag/pipeline/search.py index f791605..ca7e97b 100644 --- a/learn2rag/pipeline/search.py +++ b/learn2rag/pipeline/search.py @@ -409,7 +409,7 @@ def search_multi(multi_query: dict[str, str], user_config: dict[str, Any], opt_c return results -async def search_authorized(question: str, user: str, *, request_id: str | None = None) -> List[ScoredPoint]: +async def search_authorized(question: str, user: str, *, request_id: str | None = None, user_config = user_config, opt_config = opt_config) -> List[ScoredPoint]: points = _collect_query_points(question, user_config, opt_config, request_id=request_id) query_response = QueryResponse(points=points) authorized_points = await filter_authorized(user, query_response) From 4cae5d776c0f2fa3cf7c4fc137832d2fdbca2e62 Mon Sep 17 00:00:00 2001 From: Carolin Walter Date: Wed, 27 May 2026 13:40:15 +0000 Subject: [PATCH 14/47] make search faster --- learn2rag/pipeline/app.py | 6 ++ learn2rag/pipeline/ingestion.py | 1 + learn2rag/pipeline/qdrant.py | 103 ++++++++++++++++---------------- learn2rag/pipeline/search.py | 15 ++++- 4 files changed, 73 insertions(+), 52 deletions(-) diff --git a/learn2rag/pipeline/app.py b/learn2rag/pipeline/app.py index 44ac781..2bfedf4 100644 --- a/learn2rag/pipeline/app.py +++ b/learn2rag/pipeline/app.py @@ -15,6 +15,7 @@ from . import generate from . import ingestion from .config import user_config, opt_config +from .qdrant import Qdrant from .search import search_authorized @@ -58,6 +59,11 @@ async def simple_chatbot_response(input: QuestionInput) -> Any: app = FastAPI() +@app.on_event("startup") +async def startup_event() -> None: + Qdrant.ensure_collection(user_config["collection_name"], opt_config) + + @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse: message = str(exc) diff --git a/learn2rag/pipeline/ingestion.py b/learn2rag/pipeline/ingestion.py index 9690a28..35b2f4d 100644 --- a/learn2rag/pipeline/ingestion.py +++ b/learn2rag/pipeline/ingestion.py @@ -246,5 +246,6 @@ def index(documents: list[Document], user_config: dict[str, Any], opt_config: di opt_config (dict[str, Any]): Optimisation configuration dict. """ collection_name = user_config["collection_name"] + Qdrant.ensure_collection(collection_name=collection_name, opt_config=opt_config) qdrant = Qdrant(collection_name=collection_name, opt_config=opt_config) ingest_batch(documents, qdrant, user_config, opt_config) \ No newline at end of file diff --git a/learn2rag/pipeline/qdrant.py b/learn2rag/pipeline/qdrant.py index 3c1acb4..4939d4a 100644 --- a/learn2rag/pipeline/qdrant.py +++ b/learn2rag/pipeline/qdrant.py @@ -22,56 +22,59 @@ def __init__(self, collection_name: str, opt_config: dict[str, Any]) -> None: self.query_mode = opt_config["query_mode"] self.multi_search = opt_config["multi_search"] - if self.search_mode == "dense_sparse": - if not Qdrant.client.collection_exists(self.collection_name): - Qdrant.client.create_collection( - collection_name=self.collection_name, - vectors_config={ - "dense": VectorParams(size=self.vector_size, distance=Distance.COSINE) - }, - sparse_vectors_config={ - "sparse": SparseVectorParams( - index=SparseIndexParams(on_disk=False) - ), - }, - ) - elif self.search_mode == "dense_sparse_colbert": - if not Qdrant.client.collection_exists(self.collection_name): - Qdrant.client.create_collection( - collection_name=self.collection_name, - vectors_config={ - "dense": VectorParams(size=self.vector_size, distance=Distance.COSINE), - "colbert": VectorParams( - size=self.vector_size, - distance=Distance.COSINE, - multivector_config=MultiVectorConfig( - comparator=MultiVectorComparator.MAX_SIM, - ) - ), - }, - sparse_vectors_config={ - "sparse": SparseVectorParams( - index=SparseIndexParams(on_disk=False) - ), - }, - - ) - elif self.query_mode == "multi": - if not Qdrant.client.collection_exists(self.collection_name): - vector_size = (len(self.multi_search)+1)*self.vector_size - Qdrant.client.create_collection( - collection_name=self.collection_name, - vectors_config={ - "multi": VectorParams(size=vector_size, distance=Distance.COSINE) - } - ) + @classmethod + def ensure_collection(cls, collection_name: str, opt_config: dict[str, Any]) -> None: + if cls.client.collection_exists(collection_name): + return + vector_size = opt_config["vector_size"][opt_config["embedding_model"]] + search_mode = opt_config["search_mode"] + query_mode = opt_config["query_mode"] + multi_search = opt_config["multi_search"] + if search_mode == "dense_sparse": + cls.client.create_collection( + collection_name=collection_name, + vectors_config={ + "dense": VectorParams(size=vector_size, distance=Distance.COSINE) + }, + sparse_vectors_config={ + "sparse": SparseVectorParams( + index=SparseIndexParams(on_disk=False) + ), + }, + ) + elif search_mode == "dense_sparse_colbert": + cls.client.create_collection( + collection_name=collection_name, + vectors_config={ + "dense": VectorParams(size=vector_size, distance=Distance.COSINE), + "colbert": VectorParams( + size=vector_size, + distance=Distance.COSINE, + multivector_config=MultiVectorConfig( + comparator=MultiVectorComparator.MAX_SIM, + ) + ), + }, + sparse_vectors_config={ + "sparse": SparseVectorParams( + index=SparseIndexParams(on_disk=False) + ), + }, + ) + elif query_mode == "multi": + multi_vector_size = (len(multi_search) + 1) * vector_size + cls.client.create_collection( + collection_name=collection_name, + vectors_config={ + "multi": VectorParams(size=multi_vector_size, distance=Distance.COSINE) + } + ) else: - if not Qdrant.client.collection_exists(self.collection_name): - Qdrant.client.create_collection( - collection_name=self.collection_name, - vectors_config={ - "dense": VectorParams(size=self.vector_size, distance=Distance.COSINE) - } - ) + cls.client.create_collection( + collection_name=collection_name, + vectors_config={ + "dense": VectorParams(size=vector_size, distance=Distance.COSINE) + } + ) diff --git a/learn2rag/pipeline/search.py b/learn2rag/pipeline/search.py index ca7e97b..934affc 100644 --- a/learn2rag/pipeline/search.py +++ b/learn2rag/pipeline/search.py @@ -2,6 +2,7 @@ from typing import List, Any, cast import logging import copy +from functools import lru_cache import numpy as np from FlagEmbedding import FlagReranker # type: ignore[import-untyped] @@ -19,6 +20,16 @@ profilingLogger = logging.getLogger('profiling') +@lru_cache(maxsize=4) +def _get_flag_reranker(model_name: str, use_fp16: bool) -> FlagReranker: + return FlagReranker(model_name, use_fp16=use_fp16) + + +@lru_cache(maxsize=4) +def _get_cross_encoder(model_name: str) -> CrossEncoder: + return CrossEncoder(model_name) + + def _sort_and_deduplicate(points: list[ScoredPoint]) -> list[ScoredPoint]: best_by_id: dict[str, ScoredPoint] = {} fallback_points: list[ScoredPoint] = [] @@ -48,7 +59,7 @@ def _rerank_points_with_flagreranker( model_name: str = "BAAI/bge-reranker-v2-m3", use_fp16: bool = True, ) -> list[ScoredPoint]: - reranker = FlagReranker(model_name, use_fp16=use_fp16) + reranker = _get_flag_reranker(model_name, use_fp16) valid_points = [p for p in points if p.payload and isinstance(p.payload.get("content"), str)] if len(valid_points) != len(points): @@ -78,7 +89,7 @@ def _rerank_points_with_sentence_transformers( model_name: str = "cross-encoder/ms-marco-MiniLM-L6-v2", ) -> list[ScoredPoint]: - model = CrossEncoder(model_name) + model = _get_cross_encoder(model_name) valid_points = [p for p in points if p.payload and isinstance(p.payload.get("content"), str)] if len(valid_points) != len(points): From 38d090b2b5ab0dd07a258cbe2858b61196e9c921 Mon Sep 17 00:00:00 2001 From: Carolin Walter Date: Wed, 27 May 2026 13:42:49 +0000 Subject: [PATCH 15/47] hack to ingest kcenter dump with new updater mechanism --- learn2rag/pipeline/main.py | 60 ++++++++-------- learn2rag/pipeline/old_loaders.py | 109 ++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 27 deletions(-) create mode 100644 learn2rag/pipeline/old_loaders.py diff --git a/learn2rag/pipeline/main.py b/learn2rag/pipeline/main.py index ca7a199..03b6aae 100755 --- a/learn2rag/pipeline/main.py +++ b/learn2rag/pipeline/main.py @@ -4,6 +4,8 @@ from langchain_core.documents.base import Document +from . import old_loaders + from . import ingestion from . import search from . import generate @@ -19,7 +21,7 @@ from .config import user_config, opt_config #delete_collection(loader_id="json_test_file", user_config=user_config, opt_config=opt_config) - results = get_documents(loader_id="json_test_file", user_config=user_config, opt_config=opt_config) + #results = get_documents(loader_id="json_test_file", user_config=user_config, opt_config=opt_config) documents = [ Document(page_content=d["content"], metadata=d["metadata"]) @@ -33,7 +35,7 @@ "process_date": "2025-07-28", "process_time": "14:42:02", "loader_type": "DirectoryLoader", - "loader_id": "json_test_file", + "loader_id": "local_docs", "title": "The title of a real document", "summary": "This document is awesome" }, @@ -48,36 +50,40 @@ "process_date": "2025-07-28", "process_time": "14:42:02", "loader_type": "DirectoryLoader", - "loader_id": "json_test_file", + "loader_id": "local_docs", "title": "The title of a real document", "summary": "This document is awesome" }, - "content": "Qdrant ist eine Open-Source-Vektordatenbank..." + "content": "ein fortlaufender Text ohne Absätze, der sehr deutlich über 4000 Token liegen sollte: In einer weit entfernten Region, in der die Landschaft aus sanften Hügeln, dichten Wäldern, breiten Flüssen und alten Steinen bestand, lag eine Stadt, die von den Bewohnern seit Generationen als Ort des Wissens, der Geduld und der sorgfältigen Beobachtung beschrieben wurde, denn dort hatte man gelernt, dass alles, was Bestand haben soll, nicht nur gebaut, sondern auch verstanden werden muss, und so begann jeder Tag mit einem stillen Blick auf das, was sich verändert hatte, sei es das Licht auf den Dächern, der Wind in den Gassen, das Wasser an den Kaimauern oder die Art, wie Menschen miteinander sprachen, wenn sie glaubten, ungehört zu sein; in dieser Stadt lebte ein Archivarin namens Mara, die jeden Morgen vor Sonnenaufgang durch die schmalen Straßen zum Haus der Aufzeichnungen ging, einem Gebäude aus dunklem Holz und hellem Stein, dessen Fenster so hoch waren, dass sie den Himmel wie schmale Streifen einfassten, und während andere noch schliefen, ordnete Mara Pergamente, Karten, Briefe, Berichte, Listen und Fragmente von Erzählungen, weil sie überzeugt war, dass eine Gesellschaft nur dann ihre Zukunft verstehen konnte, wenn sie ihre Vergangenheit nicht als Last, sondern als Werkzeug betrachtete, und jedes Dokument, das durch ihre Hände ging, war für sie nicht nur Papier oder Tinte, sondern ein Hinweis auf Entscheidungen, Zweifel, Hoffnungen und Versprechen, die irgendwann einmal in die Welt gesetzt worden waren; an den Tagen, an denen der Markt besonders belebt war, hörte sie vom offenen Fenster ihres Arbeitsraums das Rufen der Händler, das Klappern der Wagenräder und das leise Murmeln der Kunden, die Obst, Stoffe, Werkzeuge, Gewürze oder Bücher kauften, und obwohl sie selten selbst hinunterging, kannte sie die Händler nach ihren Stimmen, ihren Gewohnheiten und den kleinen Geschichten, die sie beim Liefern erzählten, etwa von einem Fischer, der behauptete, ein silberner Fisch habe ihm den Weg durch dichten Nebel gezeigt, oder von einer Schneiderin, die aus Stoffresten Karten nähte, um Kinder auf spielerische Weise geographische Zusammenhänge verstehen zu lassen; Mara notierte solche Geschichten nicht, weil sie sie für wahr oder unwahr im einfachen Sinn hielt, sondern weil sie darin eine Art menschliche Logik erkannte, denn selbst Fantasie folgt Regeln, und selbst Irrtum verrät, worauf jemand gehofft oder wovor er sich gefürchtet hat, und so lernte sie, zwischen offenkundigen Fakten und inneren Wahrheiten zu unterscheiden, ohne eines von beiden geringzuschätzen; eines Tages brachte der Bote des Stadtrats eine Mappe mit roten Schnüren, versiegelt mit dem Zeichen der sechs Türme, und darin befand sich eine Nachricht, die den ganzen Rhythmus der Stadt verändern sollte, weil sie ankündigte, dass der alte Wasserkanal, der seit Jahrhunderten die Gärten versorgte und die Mühlen antrieb, in den kommenden Monaten neu vermessen werden müsse, da sein Lauf sich an mehreren Stellen verschoben habe und die Gefahr bestehe, dass nach starkem Regen Teile des Ufers brechen könnten, wodurch nicht nur Felder, sondern auch Wohnviertel bedroht wären; viele sahen in dieser Ankündigung zunächst bloß eine technische Angelegenheit, doch Mara erkannte sofort, dass es in Wahrheit eine Prüfung des kollektiven Gedächtnisses war, denn die letzten vollständigen Karten des Kanals stammten aus einer Zeit, in der andere Messmethoden verwendet worden waren, und manche Namen der Uferabschnitte existierten nur noch in den Erzählungen alter Menschen, die sich an Überschwemmungen, Reparaturen und Umleitungen erinnerten, die nie offiziell verzeichnet worden waren; also begann sie, gemeinsam mit einem kleinen Kreis aus Gelehrten, Handwerkern, Wasserbauern, Schiffern und Anwohnern ein System des Sammelns zu entwerfen, das nicht nur auf amtlichen Dokumenten beruhte, sondern auch auf mündlicher Überlieferung, auf Skizzen, auf Markierungen an Mauern, auf eingeritzten Zeichen an Steinen und auf den Erinnerungen jener, die das Wasser jahrelang beobachtet hatten, und während sie arbeiteten, wurde deutlich, dass die Stadt nicht an Informationsmangel litt, sondern an der Zerstreuung ihres Wissens, denn vieles war vorhanden, aber nichts sprach mit dem anderen, und so mussten Brücken nicht nur über Flüsse, sondern auch zwischen Menschen, Disziplinen und Gewohnheiten gebaut werden; Mara selbst war keine große Rednerin, doch sie stellte die Fragen, die sonst niemand zu stellen wagte, zum Beispiel, warum bestimmte Berichte immer wieder dieselben Formulierungen benutzten, obwohl die Situation vor Ort anders war, oder weshalb eine Entscheidung, die auf einem alten Erlass beruhte, nie an die veränderten Gegebenheiten angepasst worden war, und häufig stellte sich heraus, dass niemand aus Böswilligkeit geschwiegen hatte, sondern aus Routine, aus Respekt vor Zuständigkeiten oder aus der stillen Annahme, jemand anders werde das Problem schon bemerken, bevor es ernst werde; genau darin lag für Mara die größte Gefahr jeder gut organisierten Gemeinschaft, dass nämlich die Annahme von Ordnung dazu verleiten kann, kritische Fragen zu vermeiden, weil man die Existenz von Formularen, Zuständigkeiten und Protokollen bereits mit echter Verständigung verwechselt, und deshalb bestand ihre Arbeit nicht nur aus dem Sammeln von Daten, sondern aus dem Einüben einer Kultur des Nachfragens, des Abgleichs und der gemeinsamen Prüfung; während die Vermessung des Kanals begann, begleitete sie eine Gruppe von Ingenieuren, die lange Stäbe, Messketten, Notiztafeln und kleine Wetterinstrumente trugen, und die ersten Tage waren von Präzision geprägt, von Zahlenreihen, Höhenangaben und Abständen, doch bald stießen sie auf Widersprüche zwischen Karte und Realität, zwischen Bericht und Gelände, zwischen offizieller Bezeichnung und lokalem Sprachgebrauch, denn an einer Stelle sprach der eine vom „alten Knick“, der nächste vom „Fischgrund“, ein dritter von der „Schattenbiegung“, und jedes Mal war es dieselbe Stelle, nur aus einer anderen Erinnerung heraus benannt, sodass Mara vorschlug, nicht einen Namen zu wählen und die übrigen zu verwerfen, sondern alle Namen nebeneinander zu führen, weil jeder von ihnen einen Blickwinkel enthielt, der für andere Zwecke nützlich sein konnte; diese Entscheidung erwies sich später als entscheidend, als ein Regenband aus dem Norden kam und die Wasserstände rascher stiegen als erwartet, denn die Menschen entlang des Kanals erkannten an den verschiedenen Namen sofort, welche Abschnitte jeweils gemeint waren, obwohl in früheren amtlichen Listen nur eine einzige Bezeichnung gestanden hatte, die für manche Bewohner kaum etwas sagte; so zeigte sich, dass Sprache nicht bloß etikettiert, sondern Orientierung schafft, und je genauer eine Gemeinschaft hinsieht, desto eher versteht sie, dass Mehrdeutigkeit nicht immer ein Problem ist, sondern manchmal ein Speicher von Erfahrung; in den Wochen danach wuchs Maras Arbeit über den Kanal hinaus, denn aus der anfänglichen Vermessung entwickelte sich ein umfassenderes Vorhaben, das die Stadt selbst betraf, ihre Lagerhäuser, ihre Straßen, ihre Werkstätten, ihre Schulen und ihre Archive, weil alle erkannten, dass die Frage, wie man Wissen ordnet, eng mit der Frage verknüpft ist, wie man Verantwortung verteilt, und so bildeten sich kleine Gruppen, die Themen wie Wartung, Beschaffung, Kommunikation, Dokumentation und Krisenreaktion neu organisierten, wobei jede Gruppe darauf achten sollte, nicht nur Berichte zu produzieren, sondern auch Rückmeldungen zu erhalten, Fehler offen zu benennen und Zuständigkeiten so zu formulieren, dass andere sie verstehen konnten; es war kein leichter Prozess, denn Menschen fürchten oft nicht die Arbeit selbst, sondern die Veränderung ihrer Gewohnheiten, und manche Beamte empfanden es als Angriff, wenn man sie bat, ihre alten Abläufe zu erklären, während einige Handwerker misstrauisch wurden, wenn ihre praktischen Kenntnisse plötzlich in Tabellen auftauchten, doch Mara vermittelte stets, dass Sichtbarkeit nicht Kontrolle bedeute, sondern Zusammenarbeit, und dass ein guter Prozess nicht nur denjenigen nützt, die ihn aufsetzen, sondern allen, die auf seine Ergebnisse angewiesen sind; in einer besonders langen Sitzung, die bis in die Abenddämmerung dauerte, wurde über die Frage gestritten, ob man für jede wichtige Entscheidung eine zentrale Stelle bestimmen solle oder ob es besser sei, mehrere unabhängige Bestätigungen einzuholen, und Mara hörte still zu, bis die Stimmen müde wurden, dann sagte sie, dass Zentralisierung dort sinnvoll sei, wo Zeit knapp und Risiko hoch sei, während Verteilung dort Vorteile biete, wo Vielfalt an Beobachtungen nötig sei, und dass man beides nicht als Gegensätze, sondern als Werkzeuge verstehen müsse, die je nach Lage unterschiedlich einzusetzen seien; dieser Satz wurde später oft wiederholt, nicht weil er besonders poetisch gewesen wäre, sondern weil er eine Haltung zusammenfasste, die viele erst allmählich lernten: dass Systeme nur dann robust werden, wenn sie sowohl klare Verantwortlichkeit als auch Raum für Korrektur besitzen; währenddessen lebte außerhalb der Stadt ein alter Kartograph namens Elian, der einst für den Hof gearbeitet hatte und nun in einem kleinen Haus am Waldrand lebte, umgeben von Kartenrollen, Holzlinealen, Glaslinsen und einem Ofen, der im Winter den Raum trocken hielt, und als er von Maras Arbeit hörte, schickte er ihr eine Nachricht mit der Einladung, seine Sammlung alter Flussverläufe zu prüfen, weil er vermutete, dass darin Hinweise auf frühere Umleitungen verborgen waren, die in offiziellen Dokumenten nicht mehr vorkamen; Mara reiste zu ihm an einem Morgen, an dem dichter Nebel über den Wiesen lag, und fand einen Mann, dessen Hände von der jahrelangen Arbeit an Papier und Pergament leicht gefärbt waren, aber dessen Augen noch immer die lebhafte Aufmerksamkeit eines Suchenden hatten, und er zeigte ihr Karten aus verschiedenen Jahrzehnten, manche mit feinen Bleistiftlinien, manche mit Tinte, manche mit verblassten Markierungen, die er anhand von Randnotizen, Wasserständen und lokalen Erzählungen datiert hatte, und gemeinsam entdeckten sie, dass mehrere Fehlstellen in den aktuellen Aufzeichnungen nicht auf Zufall beruhten, sondern auf der Praxis früherer Schreiber, störende Ereignisse zu glätten, um ein harmonischeres Bild der Stadt zu hinterlassen; diese Erkenntnis erschütterte Mara nicht, denn sie hatte schon lange geahnt, dass jede Verwaltung dazu neigt, das Unbequeme zu verkleinern, wenn nicht gar zu verschweigen, doch sie war dankbar für den Beleg, weil er ihr half, andere davon zu überzeugen, dass Offenheit kein Zeichen von Schwäche ist, sondern die Voraussetzung für belastbare Entscheidungen; Elian wiederum sagte, dass eine Karte niemals die Welt selbst sei, sondern ein Vertrag zwischen Gedächtnis und Gebrauch, und diese Formulierung schrieb Mara später über ihr Bett, weil sie ihr half, die Grenzen jeder Darstellung zu respektieren, sei es eine Karte, ein Bericht, ein Diagramm, ein Protokoll oder ein Modell; im Verlauf des Sommers wurden die Arbeiten am Kanal vollendet, doch daraus entstand kein Gefühl des Abschlusses, sondern vielmehr ein neues Bewusstsein dafür, dass jede Lösung weitere Fragen eröffnet, denn nun wollte man wissen, wie die Gärten effizienter bewässert werden könnten, wie die Mühlen ihre Last besser verteilen sollten, wie Schiffe ihre Routen sicherer planen könnten und wie die Bevölkerung im Falle künftiger Unwetter schneller informiert werden könnte, und so begann man, ein Netzwerk aus Signalpunkten aufzubauen, das aus Türmen, Glocken, Flaggen und Boten bestand, ergänzt durch einfache Regeln, welche Meldung welchen Weg nehmen sollte; Mara bestand darauf, dass die Regeln kurz, verständlich und testbar sein müssten, weil lange Anweisungen in Stresssituationen versagen, und sie führte kleine Übungen durch, bei denen absichtlich falsche oder unvollständige Nachrichten in Umlauf gebracht wurden, um zu sehen, wo das System brüchig war, und jedes Mal, wenn ein Fehler erkannt wurde, schrieb sie nicht nur die Ursache auf, sondern auch die Beobachtung, die dazu geführt hatte, denn sie wollte nicht allein wissen, was scheiterte, sondern auch, wie man das Scheitern bemerkte; nach einigen Monaten hatten sich diese Übungen zu einer Art städtischer Gewohnheit entwickelt, und selbst Kinder spielten in den Höfen, dass sie Meldungen weitergeben, prüfen und bestätigen müssten, wobei sie lachten, wenn jemand die falsche Reihenfolge wählte, aber auch lernten, dass jede Information eine Quelle und einen Zweck hat; in dieser Zeit erkrankte Elian schwer, und Mara besuchte ihn mehrfach, brachte ihm Suppe, sauberes Wasser und Abschriften seiner besten Karten, damit er trotz Schwäche weiter mitdenken konnte, und in den Gesprächen am Bett wurde deutlich, dass seine größte Sorge nicht sein eigener Zustand war, sondern die Frage, ob sein Wissen nach seinem Tod in Vergessenheit geraten würde, weil es über zu viele private Notizen verteilt war, die niemand außer ihm entschlüsseln konnte; also begannen sie gemeinsam, ein übersichtliches Verzeichnis seiner Karten und Notizen anzulegen, mit klaren Titeln, Datierungen, Querverweisen und kurzen Erläuterungen, sodass auch andere damit arbeiten konnten, und als er die ersten Ergebnisse sah, lächelte er und meinte, das sei die freundlichste Form der Unsterblichkeit, nicht als Denkmal, sondern als nutzbares Wissen weiterzuleben; Mara behielt diesen Satz lange im Gedächtnis, weil er ihr vor Augen führte, dass gute Organisation nicht nur Effizienz erzeugt, sondern Würde verleiht, da sie verhindert, dass Erkenntnisse mit ihren Urhebern sterben; als der Herbst kam, färbten sich die Bäume goldbraun, und die Stadt bereitete sich auf die kälteren Monate vor, indem Vorräte geprüft, Dächer geflickt, Leinen verstaut und Brennholz gestapelt wurden, und wieder zeigte sich, wie sehr die Qualität alltäglicher Vorbereitung von verlässlicher Information abhängt, denn wer zu spät erfährt, dass ein Dach undicht ist, verliert mehr als nur Zeit, und wer zu spät erkennt, dass ein Lager knapp wird, handelt unter Druck und macht Fehler; deshalb setzte Mara sich dafür ein, dass alle wichtigen Beobachtungen nicht erst in großen Berichten, sondern unmittelbar und in einfacher Form erfasst wurden, am besten dort, wo die Arbeit stattfand, weil jede zusätzliche Übertragung die Wahrscheinlichkeit von Missverständnissen erhöht; diese Praxis stieß anfangs auf Widerstand, weil einige meinten, es sei unnötig, Handwerker mit Listen zu belasten, doch bald stellte sich heraus, dass gerade die direkte Erfassung viele spätere Rückfragen ersparte, und so entstanden kleine, robuste Abläufe, in denen ein Werkstattmeister, eine Lagerverwalterin oder ein Botenführer sofort melden konnte, was relevant war, ohne erst durch mehrere Hierarchieebenen zu gehen; im Winter, wenn Schnee die Dächer bedeckte und die Straßen leiser wurden, kehrte die Stadt in eine langsamere, nachdenklichere Haltung zurück, und Mara nutzte die Ruhe, um aus den vielen gesammelten Notizen ein Lehrbuch des städtischen Wissens zu erstellen, das nicht nur erklärte, wie man Daten sammelt, sondern auch, wie man Zweifel formuliert, Prioritäten setzt, Fehler erkennt, Rückmeldungen verarbeitet und Entscheidungen anpasst; sie schrieb darin über die Notwendigkeit klarer Begriffe, über die Vorteile mehrfacher Perspektiven, über die Grenzen von Annahmen und über die Kunst, aus unvollständigen Informationen dennoch vernünftig zu handeln, und je weiter sie schrieb, desto mehr merkte sie, dass sie nicht bloß ein Buch zusammenstellte, sondern eine Haltung überlieferte, die vielleicht in künftigen Krisen nützlicher sein würde als jedes einzelne Detail; als das Jahr sich dem Ende näherte, kam der Stadtrat zusammen, um die Ergebnisse zu bewerten, und statt einer feierlichen Rede erwarteten sie Zahlen, Karten, Fallbeispiele und Erfahrungen aus der Praxis, und Mara präsentierte ruhig, was erreicht worden war: weniger widersprüchliche Meldungen, schnellere Reaktionszeiten, klarere Zuständigkeiten, mehr Transparenz bei der Pflege öffentlicher Anlagen und ein wachsendes Vertrauen zwischen Verwaltung und Bevölkerung, weil man gelernt hatte, dass Beteiligung nicht bedeutet, jede Entscheidung endlos zu diskutieren, sondern die richtigen Menschen zur richtigen Zeit mit den richtigen Informationen zu versorgen; die Ratsmitglieder nickten, einige überrascht, andere erleichtert, und einer von ihnen fragte, was die wichtigste Lehre des Jahres sei, worauf Mara antwortete, dass Ordnung nur dann nützlich sei, wenn sie der Wirklichkeit dient, und dass jede Struktur, die nicht regelmäßig überprüft wird, irgendwann zur bloßen Gewohnheit erstarrt; nach dieser Sitzung ging sie spät in der Nacht durch die schneebedeckten Straßen nach Hause, und während der Mond auf den Dächern lag, dachte sie daran, wie seltsam und kostbar menschliche Gemeinschaften sind, wie viel Mühe es kostet, Vertrauen aufzubauen, wie leicht Missverständnisse entstehen und wie viel sich zum Guten wendet, wenn Menschen bereit sind, einander zuzuhören, ihre Aufzeichnungen zu teilen, ihre Annahmen zu prüfen und die Welt nicht nur so zu sehen, wie sie erscheinen möchte, sondern so, wie sie tatsächlich ist; sie öffnete die Tür ihres Hauses, legte den Mantel ab, setzte Wasser auf und blickte noch einmal auf die Schachtel mit Karten, Notizen und Berichten, die auf dem Tisch stand, und empfand keine Erschöpfung, sondern stille Zuversicht, weil sie wusste, dass alles Wichtige nicht in einem einzigen großen Akt geschieht, sondern in vielen kleinen, wiederholten Gesten von Sorgfalt, Klarheit und Respekt, die zusammen etwas schaffen, das größer ist als die Summe seiner Teile, und genau darin, dachte sie, liegt die eigentliche Arbeit jeder Stadt, jeder Institution und jeder Gemeinschaft: das Verstreute zu verbinden, das Unsichere zu prüfen, das Vorläufige zu verbessern und das Vergängliche so aufzuschreiben, dass daraus Orientierung entstehen kann, heute, morgen und in den Jahren, die noch kommen werden." }, ] ] - update_documents(loader_id="json_test_file", documents=documents, user_config=user_config, opt_config=opt_config) - ingestion.index(documents, user_config, opt_config) - - if opt_config["query_mode"] == "multi": - # in query_mode 'multi' different querys for each vector in the multi-vector are allowed - multi_query = {"content": "What is USM AI?", "title": "What is USM AI?", "summary": "What is USM AI?", "source_path":"USU/ITSM/"} - results = search.search_multi(multi_query, user_config, opt_config, request_id=None) - points = results.points - # modify the query for generation part - query = " ".join(f"{k}={v}" for k, v in multi_query.items()) - else: - query = "Was sind A, B und C?" - user = "anonymous" - points = asyncio.run(search.search_authorized(query, user, request_id=None)) - - sources = "\n".join(set(point.payload['path'] for point in points)) # type: ignore[index] - - for point in points: - print(f"ID: {point.id}, Path: {point.payload['path']}, Score: {point.score}") # type: ignore[index] + #update_documents(loader_id="json_test_file", documents=documents, user_config=user_config, opt_config=opt_config) + + all_documents = old_loaders.json_loader(user_config['imported_documents_file_path']) + ingestion.index(all_documents, user_config, opt_config) - answer = generate.generate(query, points, opt_config) + print("finished ingestion") - print(query) - print(answer) - print(sources) + #if opt_config["query_mode"] == "multi": + # # in query_mode 'multi' different querys for each vector in the multi-vector are allowed + # multi_query = {"content": "What is USM AI?", "title": "What is USM AI?", "summary": "What is USM AI?", "source_path":"USU/ITSM/"} + # results = search.search_multi(multi_query, user_config, opt_config, request_id=None) + # points = results.points + # # modify the query for generation part + # query = " ".join(f"{k}={v}" for k, v in multi_query.items()) + #else: + # query = "Was sind A, B und C?" + # user = "anonymous" + # points = asyncio.run(search.search_authorized(query, user, request_id=None)) +# + #sources = set(point.payload['source'] for point in points) # type: ignore[index] +# + #for point in points: + # print(f"ID: {point.id}, Source: {point.payload['source']}, Score: {point.score}") # type: ignore[index] +# + #answer = generate.generate(query, points, opt_config) +# + #print(query) + #print(answer) + #print(sources) diff --git a/learn2rag/pipeline/old_loaders.py b/learn2rag/pipeline/old_loaders.py new file mode 100644 index 0000000..eacaaa7 --- /dev/null +++ b/learn2rag/pipeline/old_loaders.py @@ -0,0 +1,109 @@ +import asyncio +import bz2 +import itertools +import logging +from typing import Iterator, Sequence +import os + +from bs4 import SoupStrainer # type: ignore[attr-defined] +from lxml import etree + +from langchain_community.document_loaders import ( + BSHTMLLoader, + PyPDFLoader, + WebBaseLoader, + JSONLoader, +) +from langchain_core.documents import Document + + +def json_loader(file_path: str) -> list[Document]: + loader = JSONLoader( + file_path, + jq_schema=".[]", + content_key="content", + metadata_func=lambda record, meta: record.get("metadata", {}), + ) + return loader.load() + + +def html_loader(file_path: str) -> list[Document]: + loader = BSHTMLLoader(file_path) + return loader.load() + + +def pdf_loader(file_path: str) -> list[Document]: + loader = PyPDFLoader(file_path) + docs = asyncio.run(loader.aload()) + return docs + + +def sync_pdf_loader(file_path: str) -> list[Document]: + all_documents = [] + for file in os.listdir(file_path): + if file.endswith(".pdf"): + loader = PyPDFLoader( + os.path.join(file_path, file), mode="single" + ) # important: default of mode is page-vise! + docs = loader.load() + all_documents.extend(docs) + return all_documents + + +def web_loader(web_path: Sequence[str]) -> list[Document]: + bs4_strainer = SoupStrainer( + class_=[ + "SP-Content__main", + "section sectionZ sectionArticle", + "page__content", + "post-content", + ] + ) + loader = WebBaseLoader( + web_path=web_path, + bs_kwargs={"parse_only": bs4_strainer}, + ) + docs = loader.load() + return docs + + +def cleanup_etree(elem: etree._Element) -> None: + elem.clear() + while elem.getprevious() is not None: + del elem.getparent()[0] # type:ignore[union-attr] + + +def read_wikibooks_dump(path: str) -> Iterator[tuple[str, str]]: + total = 0 + skipped = 0 + file = bz2.open(path) + for action, elem in etree.iterparse( + file, events=("end",), recover=True, huge_tree=True + ): + localname = etree.QName(elem).localname + if localname == "page": + title = elem.xpath('*[local-name()="title"]') + text = elem.xpath('*//*[local-name()="text"]') + if len(title) == 1 and len(text) == 1 and text[0].text is not None: + yield title[0].text, text[0].text + total += 1 + if total % 10000 == 0: + logging.debug("Pages read from %s: %d", path, total) + else: + skipped += 1 + cleanup_etree(elem) + logging.debug("Pages skipped in %s: %d", path, skipped) + + +def wikibooks_loader(path: str, limit: int | None = None) -> list[Document]: + docs: list[Document] = [] + for title, text in itertools.islice(read_wikibooks_dump(path), limit): + docs.append( + Document( + page_content=text, + metadata={ + "source": f"wikibooks:{title}", + }, + ) + ) + return docs \ No newline at end of file From 2f6e15fb951029babd1c09624f7ba41fa7590745 Mon Sep 17 00:00:00 2001 From: Hanna Reder Date: Fri, 29 May 2026 08:11:08 +0000 Subject: [PATCH 16/47] include time in objective function & minor fixes --- learn2rag/evaluation/tools.py | 18 +++-- .../optimization/retrieval_optimization.py | 72 +++++++++++-------- learn2rag/pipeline/search.py | 9 ++- 3 files changed, 62 insertions(+), 37 deletions(-) diff --git a/learn2rag/evaluation/tools.py b/learn2rag/evaluation/tools.py index ea1e351..307e4df 100644 --- a/learn2rag/evaluation/tools.py +++ b/learn2rag/evaluation/tools.py @@ -80,21 +80,27 @@ def ingest_dataset_documents(dataset_name: str) -> None: # learn2rag.pipeline.ingestion.index(user_config, opt_config) -def read_dataset_qa(dataset_name: str, subdirectory: str, split: str | None=None) -> Any: +def read_dataset_qa(dataset_name: str, subdirectory: str, split: str | None = None) -> Any: logging.debug(f'{dataset_name=}') if subdirectory.endswith('.csv'): df = pd.read_csv(subdirectory, sep=';') # wrap in object that behaves like HuggingFace dataset class CSVDataset: - def __init__(self, df): + data: pd.DataFrame + + def __init__(self, df: pd.DataFrame) -> None: self.data = df - def __len__(self): + + def __len__(self) -> int: return len(self.data) - def __getitem__(self, idx): - return self.data.iloc[idx].to_dict() - def select(self, indices): + + def __getitem__(self, idx: int) -> dict[str, Any]: + return dict(self.data.iloc[idx].to_dict()) + + def select(self, indices: Any) -> "CSVDataset": return CSVDataset(self.data.iloc[list(indices)]) + return CSVDataset(df) else: diff --git a/learn2rag/optimization/retrieval_optimization.py b/learn2rag/optimization/retrieval_optimization.py index 77095b5..6a1184a 100644 --- a/learn2rag/optimization/retrieval_optimization.py +++ b/learn2rag/optimization/retrieval_optimization.py @@ -10,46 +10,44 @@ import copy import os import asyncio -from typing import Dict, Any, List, Union, Tuple +from typing import Dict, Any, List, Union, Tuple, cast import numpy as np -from bert_score import score as bert_score +# from bert_score import score as bert_score # type: ignore[import-not-found] from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation, Configuration, \ - ForbiddenAndConjunction, ForbiddenEqualsClause -from smac import HyperparameterOptimizationFacade, Scenario + ForbiddenAndConjunction, ForbiddenEqualsClause # type: ignore[import-not-found] +from smac import HyperparameterOptimizationFacade, Scenario # type: ignore[import-not-found] from learn2rag.evaluation.tools import read_dataset_qa from learn2rag.pipeline.config import opt_config import learn2rag.pipeline.search -import learn2rag.pipeline.generate +# import learn2rag.pipeline.generate -def load_registry(path: str = "registry.json") -> dict: +def load_registry(path: str = "registry.json") -> dict[str, Any]: p = pathlib.Path(path) if not p.is_file(): logging.error("registry file not found") with p.open() as f: - return json.load(f) + return cast(dict[str, Any], json.load(f)) def run_search(question: str, user_config: Dict[str, Any], working_config: Dict[str, Any]) -> Tuple[List[Any], float]: t0 = time.time() - # docs = learn2rag.pipeline.search.search(question, user_config, working_config) - # TODO user config/opt config docs = asyncio.run(learn2rag.pipeline.search.search_authorized(question, user="anonymous", request_id=None, user_config=user_config, opt_config=working_config)) search_time = time.time() - t0 - source_list = [point.payload['source'] for point in docs] + source_list = [point.payload['source'] for point in docs if point.payload is not None and "source" in point.payload] return source_list, search_time -def recall(search_results, labels): +def recall(search_results: list[list[Any]], labels: list[Any]) -> float: count = 0 top_k = opt_config["top_k"] for q in range(len(search_results)): label = str(labels[q]) hits = [str(h) for h in search_results[q]] - print('label ',label, ' hits: ', hits) + # print('label ', label, ' hits: ', hits) if label in hits[:top_k]: count += 1 return count / len(labels) if labels else 0.0 @@ -59,10 +57,8 @@ def recall(search_results, labels): # removed dataset_name because it just use in user config and now we inject it def objective(config: Configuration, questions: List[Dict[str, Any]], - dataset_name: str, state: Dict[str, Any], answers_dir: pathlib.Path - ,prompt_map ) -> float: state["trial_count"] += 1 tid = state["trial_count"] @@ -81,8 +77,6 @@ def objective(config: Configuration, logging.warning(f"Skip invalid cfg: {cfg}") return 1.0 - -# TODO working_cfg = copy.deepcopy(opt_config) working_cfg.update({ "chunk_size": cfg["chunk_size"], @@ -95,10 +89,9 @@ def objective(config: Configuration, "rewrite": cfg["rewrite"] }) -# TODO ucfg = { "file_path": None, - "collection_name": "CSC-CS_2000-CO_50", # TODO get collection name depending on hyperparameters + "collection_name": f"CSC-CS_{cfg['chunk_size']}-CO_{cfg['chunk_overlap']}", "imported_documents_file_path": None, "llm": None, } @@ -106,7 +99,7 @@ def objective(config: Configuration, if env_user_cfg and pathlib.Path(env_user_cfg).exists(): ucfg.update(json.loads(pathlib.Path(env_user_cfg).read_text())) - predictions, goldens = [], [] + predictions, goldens = [], [] # type: ignore qa_pairs = [] t_start = time.time() t_search = 0.0 @@ -134,16 +127,23 @@ def objective(config: Configuration, t_score = time.time() recall_score = recall(predictions, goldens) scoring_time = time.time() - t_score + total_time = time.time() - t_start # objective function - cost = 1.0 - recall_score - total_time = time.time() - t_start + w_recall = 0.5 + w_time = 0.5 + t_search_per_sample_upper = 50 + max_time_s = t_search_per_sample_upper*len(predictions) + time_cost = max(0.0, 1.0 - (t_search / max_time_s)) + cost = 1.0 - w_recall*recall_score - w_time*time_cost + avg_t_search = t_search/len(predictions) trial_answers = { "trial_id": tid, "config": cfg, "cost": float(cost), - "avg_recall": float(recall_score), + "recall": float(recall_score), + "avg_t_search": float(avg_t_search), "qa_pairs": qa_pairs, } answers_file = answers_dir / f"trial_{tid}_answers.json" @@ -154,14 +154,16 @@ def objective(config: Configuration, state["convergence"].append({"trial": tid, "cost": float(cost), "best_cost": float(state["best_cost"])}) state["history"].append({ "trial_id": tid, "config": cfg, - "avg_recall": float(recall_score), - "cost": float(cost), "time_s": round(total_time, 2), + "recall": float(recall_score), + "avg_t_search": float(avg_t_search), + "cost": float(cost), + "time_s": round(total_time, 2), "search_s": round(t_search, 2), "scoring_s": round(scoring_time, 2), }) logging.info( - f"Trial {tid}: recall={recall_score:.4f} cost={cost:.4f} " + f"Trial {tid}: recall={recall_score:.4f} avg_t_search={avg_t_search: .4f} cost={cost:.4f} " f"time={total_time:.1f}s (search={t_search:.1f} score={scoring_time:.1f})" ) return float(cost) @@ -186,7 +188,7 @@ def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathli total = sum(raw.values()) imp = {p: round(v / total, 4) for p, v in raw.items()} if total > 0 else raw - ranking = sorted(imp, key=imp.get, reverse=True) + ranking = sorted(imp, key=imp.get, reverse=True) # type: ignore[arg-type] result = {"method": "variance_based", "ranking": ranking, "individual": imp} with open(output_path / "parameter_importance.json", "w") as f: json.dump(result, f, indent=2) @@ -223,10 +225,8 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ cs = ConfigurationSpace(seed=42) cs.add([ - # Categorical("chunk_size", [250, 1000, 2000], default=1000), - # Categorical("chunk_overlap", [50, 200], default=50), - Categorical("chunk_size", [2000], default=2000), - Categorical("chunk_overlap", [50], default=50), + Categorical("chunk_size", [250, 1000, 2000], default=1000), + Categorical("chunk_overlap", [50, 200], default=50), Categorical("search_mode", ["dense", "sparse", "dense_sparse", "dense_sparse_colbert"], default="dense"), Categorical("reranking_mode", ["none", "reranking_with_flagreranker", "reranking_with_sentence_transformers", "reranking_with_colbert"], default="none"), Categorical("rewrite_mode", ["none", "subqueries", "keywords", "subqueries_keywords"], default="none"), @@ -236,6 +236,11 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ ]) cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) + cs.add(ForbiddenAndConjunction( + ForbiddenEqualsClause(cs["chunk_size"], 250), + ForbiddenEqualsClause(cs["chunk_overlap"], 200), + )) + for sm in ["dense", "sparse"]: for fm in ["DBSF", "RRF"]: cs.add(ForbiddenAndConjunction( @@ -292,9 +297,16 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ importance = param_importance(smac, out) total_time = time.time() - t0 best_cfg = incumbent.get_dictionary() + best_trial = min( + (h for h in state["history"] if h.get("config") == best_cfg), + key=lambda h: h["cost"], + default=None, + ) + best_trial_id = best_trial["trial_id"] if best_trial else None results_path = out / "optimization_results.json" results_path.write_text(json.dumps({ "best_config": best_cfg, + "best_trial_id": best_trial_id, "run_history": state["history"], "convergence": state["convergence"], "parameter_importance": importance, diff --git a/learn2rag/pipeline/search.py b/learn2rag/pipeline/search.py index 934affc..1ab26b7 100644 --- a/learn2rag/pipeline/search.py +++ b/learn2rag/pipeline/search.py @@ -138,6 +138,7 @@ def _rerank_points_with_colbert( query=colbert_query, # type: ignore[arg-type] using="colbert", limit=top_k, + timeout=100 ) return list(results.points) @@ -318,6 +319,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], query=query_embedding, # type: ignore[arg-type, unused-ignore] using="dense", limit=opt_config["top_k"], + timeout=100 ) elif opt_config["search_mode"] == "sparse": indices = [int(k) for k in query_embedding["lexical_weights"].keys()] # type: ignore[union-attr] @@ -327,6 +329,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], query=models.SparseVector(indices=indices, values=values), using="sparse", limit=opt_config["top_k"], + timeout=100 ) elif opt_config["search_mode"] == "dense_sparse": indices = [int(k) for k in query_embedding["lexical_weights"].keys()] # type: ignore[union-attr] @@ -347,6 +350,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], ], query=models.FusionQuery(fusion=fusion_mode), limit=opt_config["top_k"], + timeout=100 ) elif opt_config["search_mode"] == "dense_sparse_colbert": @@ -373,6 +377,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], ], query=models.FusionQuery(fusion=fusion_mode), limit=opt_config["top_k"], + timeout=100 ) elif opt_config["search_mode"] == "multi_search": @@ -381,6 +386,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], query=query_embedding, # type: ignore[arg-type, unused-ignore] using="multi", limit=opt_config["top_k"], + timeout=100 ) return results @@ -415,12 +421,13 @@ def search_multi(multi_query: dict[str, str], user_config: dict[str, Any], opt_c query=query_embedding, # type: ignore[arg-type, unused-ignore] using="multi", limit=opt_config["top_k"], + timeout=100 ) profilingLogger.info('end', extra={'activity': 'search', 'request_id': request_id}) return results -async def search_authorized(question: str, user: str, *, request_id: str | None = None, user_config = user_config, opt_config = opt_config) -> List[ScoredPoint]: +async def search_authorized(question: str, user: str, *, request_id: str | None = None, user_config: dict[str, Any] = user_config, opt_config: dict[str, Any] = opt_config) -> List[ScoredPoint]: points = _collect_query_points(question, user_config, opt_config, request_id=request_id) query_response = QueryResponse(points=points) authorized_points = await filter_authorized(user, query_response) From cd22a854d66e0f0a83c28eeeb4cb57b28333251a Mon Sep 17 00:00:00 2001 From: Hanna Reder Date: Fri, 29 May 2026 12:02:56 +0000 Subject: [PATCH 17/47] fix mypy issues --- learn2rag/evaluation/tools.py | 4 +-- .../optimization/retrieval_optimization.py | 30 ++++++++++++------- learn2rag/pipeline/search.py | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/learn2rag/evaluation/tools.py b/learn2rag/evaluation/tools.py index 307e4df..0bd6558 100644 --- a/learn2rag/evaluation/tools.py +++ b/learn2rag/evaluation/tools.py @@ -6,7 +6,7 @@ import logging import datasets import json_stream # type: ignore[import-untyped] -from typing import Any, Callable +from typing import Any, Callable, cast import pandas as pd import learn2rag.pipeline.ingestion @@ -96,7 +96,7 @@ def __len__(self) -> int: return len(self.data) def __getitem__(self, idx: int) -> dict[str, Any]: - return dict(self.data.iloc[idx].to_dict()) + return cast(dict[str, Any], self.data.iloc[idx].to_dict()) def select(self, indices: Any) -> "CSVDataset": return CSVDataset(self.data.iloc[list(indices)]) diff --git a/learn2rag/optimization/retrieval_optimization.py b/learn2rag/optimization/retrieval_optimization.py index 6a1184a..da50bc8 100644 --- a/learn2rag/optimization/retrieval_optimization.py +++ b/learn2rag/optimization/retrieval_optimization.py @@ -14,9 +14,16 @@ import numpy as np # from bert_score import score as bert_score # type: ignore[import-not-found] -from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation, Configuration, \ - ForbiddenAndConjunction, ForbiddenEqualsClause # type: ignore[import-not-found] -from smac import HyperparameterOptimizationFacade, Scenario # type: ignore[import-not-found] +from ConfigSpace import ( + ConfigurationSpace, + Integer, + Categorical, + ForbiddenGreaterThanRelation, + Configuration, + ForbiddenAndConjunction, + ForbiddenEqualsClause, +) +from smac import HyperparameterOptimizationFacade, Scenario from learn2rag.evaluation.tools import read_dataset_qa from learn2rag.pipeline.config import opt_config @@ -99,8 +106,10 @@ def objective(config: Configuration, if env_user_cfg and pathlib.Path(env_user_cfg).exists(): ucfg.update(json.loads(pathlib.Path(env_user_cfg).read_text())) - predictions, goldens = [], [] # type: ignore - qa_pairs = [] + predictions: list[list[Any]] = [] + goldens: list[Any] = [] + qa_pairs: list[dict[str, Any]] = [] + t_start = time.time() t_search = 0.0 @@ -178,11 +187,11 @@ def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathli if len(configs) < 3: return {} - raw = {} + raw: dict[str, float] = {} for p in params: - groups = {} + groups: dict[str, list[float]] = {} for c, cost in zip(configs, np.array(costs)): - groups.setdefault(str(c[p]), []).append(cost) + groups.setdefault(str(c[p]), []).append(float(cost)) means = [np.mean(g) for g in groups.values()] raw[p] = float(np.var(means)) if len(means) > 1 else 0.0 @@ -221,7 +230,6 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ } for i, r in enumerate(qa) ] - prompt_map = registry["prompts"] cs = ConfigurationSpace(seed=42) cs.add([ @@ -290,10 +298,12 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ smac = HyperparameterOptimizationFacade( scenario=scenario, - target_function=lambda config, seed=0: objective(config, questions, dataset_name, state, answers_dir,prompt_map) + target_function=lambda config, seed=0: objective(config, questions, state, answers_dir) ) t0 = time.time() incumbent = smac.optimize() + if isinstance(incumbent, list): + incumbent = incumbent[0] importance = param_importance(smac, out) total_time = time.time() - t0 best_cfg = incumbent.get_dictionary() diff --git a/learn2rag/pipeline/search.py b/learn2rag/pipeline/search.py index 1ab26b7..efdf857 100644 --- a/learn2rag/pipeline/search.py +++ b/learn2rag/pipeline/search.py @@ -27,7 +27,7 @@ def _get_flag_reranker(model_name: str, use_fp16: bool) -> FlagReranker: @lru_cache(maxsize=4) def _get_cross_encoder(model_name: str) -> CrossEncoder: - return CrossEncoder(model_name) + return CrossEncoder(model_name) # type: ignore[no-any-return] def _sort_and_deduplicate(points: list[ScoredPoint]) -> list[ScoredPoint]: From fbb402cb086fa7c59d8ed3cf79cbcd02f9da38c4 Mon Sep 17 00:00:00 2001 From: Hanna Reder Date: Tue, 9 Jun 2026 09:20:24 +0000 Subject: [PATCH 18/47] optimisation adjustments --- .../optimization/retrieval_optimization.py | 15 +++--- learn2rag/pipeline/ingestion.py | 46 ++++++++++++++++++- learn2rag/pipeline/search.py | 17 +++---- 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/learn2rag/optimization/retrieval_optimization.py b/learn2rag/optimization/retrieval_optimization.py index da50bc8..57fd69c 100644 --- a/learn2rag/optimization/retrieval_optimization.py +++ b/learn2rag/optimization/retrieval_optimization.py @@ -14,7 +14,7 @@ import numpy as np # from bert_score import score as bert_score # type: ignore[import-not-found] -from ConfigSpace import ( +from ConfigSpace import (# type: ignore[import-not-found] ConfigurationSpace, Integer, Categorical, @@ -23,7 +23,7 @@ ForbiddenAndConjunction, ForbiddenEqualsClause, ) -from smac import HyperparameterOptimizationFacade, Scenario +from smac import HyperparameterOptimizationFacade, Scenario# type: ignore[import-not-found] from learn2rag.evaluation.tools import read_dataset_qa from learn2rag.pipeline.config import opt_config @@ -139,8 +139,8 @@ def objective(config: Configuration, total_time = time.time() - t_start # objective function - w_recall = 0.5 - w_time = 0.5 + w_recall = 1 + w_time = 0 t_search_per_sample_upper = 50 max_time_s = t_search_per_sample_upper*len(predictions) time_cost = max(0.0, 1.0 - (t_search / max_time_s)) @@ -153,7 +153,10 @@ def objective(config: Configuration, "cost": float(cost), "recall": float(recall_score), "avg_t_search": float(avg_t_search), - "qa_pairs": qa_pairs, + "w_recall": w_recall, + "w_time": w_time, + "top_k": opt_config["top_k"], + "qa_pairs": qa_pairs } answers_file = answers_dir / f"trial_{tid}_answers.json" with open(answers_file, "w") as f: @@ -290,7 +293,7 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ cs, deterministic=True, n_trials=n_trials, - walltime_limit=7200, + walltime_limit=172800, #7200, seed=42, output_directory=out / "smac_output", ) diff --git a/learn2rag/pipeline/ingestion.py b/learn2rag/pipeline/ingestion.py index 35b2f4d..7dea721 100644 --- a/learn2rag/pipeline/ingestion.py +++ b/learn2rag/pipeline/ingestion.py @@ -6,6 +6,8 @@ import numpy as np import warnings from collections.abc import Iterator +from collections import deque +from time import perf_counter from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_core.documents import Document @@ -15,6 +17,13 @@ from .embeddings import create_embeddings +def _format_hhmmss(total_seconds: float) -> str: + seconds = max(0, int(total_seconds)) + hours, remainder = divmod(seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + + def get_chunks_metadata(chunks: list[Document], item: str) -> Iterator[str]: missing = 0 for chunk in chunks: @@ -136,8 +145,16 @@ def ingest_batch(docs: list[Document], qdrant: Qdrant, user_config: dict[str, An chunks = text_splitter.split_documents(docs) ingestion_batch_size = opt_config["ingestion_batch_size"] + total_batches = (len(chunks) + ingestion_batch_size - 1) // ingestion_batch_size + eta_window_size = 100 + report_every = 100 + recent_batch_durations: deque[float] = deque(maxlen=eta_window_size) + ingest_start = perf_counter() + logging.info('Creating embeddings and ingesting in batches...') - for batch_start in range(0, len(chunks), ingestion_batch_size): + + for batch_idx, batch_start in enumerate(range(0, len(chunks), ingestion_batch_size), start=1): + batch_started_at = perf_counter() batch_chunks = chunks[batch_start:batch_start + ingestion_batch_size] batch_content = [chunk.page_content for chunk in batch_chunks] batch_chunk_hash = [hashlib.md5(chunk.page_content.encode()).hexdigest() for chunk in batch_chunks] @@ -225,7 +242,34 @@ def ingest_batch(docs: list[Document], qdrant: Qdrant, user_config: dict[str, An else: insert(qdrant, collection_name, sample) + batch_duration = perf_counter() - batch_started_at + recent_batch_durations.append(batch_duration) + + if batch_idx % report_every == 0: + elapsed = perf_counter() - ingest_start + avg_batch_duration = sum(recent_batch_durations) / len(recent_batch_durations) + remaining_batches = total_batches - batch_idx + eta_seconds = avg_batch_duration * remaining_batches + progress_percent = (batch_idx / total_batches) * 100 if total_batches > 0 else 100.0 + + logging.info( + "Ingestion progress: %d/%d batches (%.2f%%), elapsed=%s, eta in %s", + batch_idx, + total_batches, + progress_percent, + _format_hhmmss(elapsed), + _format_hhmmss(eta_seconds), + ) + + total_elapsed = perf_counter() - ingest_start + logging.info( + "Ingestion finished: %d/%d batches (100.00%%), total_elapsed=%s", + total_batches, + total_batches, + _format_hhmmss(total_elapsed), + ) + def index(documents: list[Document], user_config: dict[str, Any], opt_config: dict[str, Any]) -> None: """ Ingest a list of documents — entry point for standalone pipeline operation. diff --git a/learn2rag/pipeline/search.py b/learn2rag/pipeline/search.py index efdf857..617eba7 100644 --- a/learn2rag/pipeline/search.py +++ b/learn2rag/pipeline/search.py @@ -138,7 +138,7 @@ def _rerank_points_with_colbert( query=colbert_query, # type: ignore[arg-type] using="colbert", limit=top_k, - timeout=100 + timeout=120 ) return list(results.points) @@ -253,7 +253,8 @@ def _collect_query_points( top_k=opt_config["top_k_reranker"], opt_config=opt_config ) - + else: + points = points[:opt_config["top_k"]] return points @@ -319,7 +320,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], query=query_embedding, # type: ignore[arg-type, unused-ignore] using="dense", limit=opt_config["top_k"], - timeout=100 + timeout=120 ) elif opt_config["search_mode"] == "sparse": indices = [int(k) for k in query_embedding["lexical_weights"].keys()] # type: ignore[union-attr] @@ -329,7 +330,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], query=models.SparseVector(indices=indices, values=values), using="sparse", limit=opt_config["top_k"], - timeout=100 + timeout=120 ) elif opt_config["search_mode"] == "dense_sparse": indices = [int(k) for k in query_embedding["lexical_weights"].keys()] # type: ignore[union-attr] @@ -350,7 +351,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], ], query=models.FusionQuery(fusion=fusion_mode), limit=opt_config["top_k"], - timeout=100 + timeout=120 ) elif opt_config["search_mode"] == "dense_sparse_colbert": @@ -377,7 +378,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], ], query=models.FusionQuery(fusion=fusion_mode), limit=opt_config["top_k"], - timeout=100 + timeout=120 ) elif opt_config["search_mode"] == "multi_search": @@ -386,7 +387,7 @@ def search(query: str, user_config: dict[str, Any], opt_config: dict[str, Any], query=query_embedding, # type: ignore[arg-type, unused-ignore] using="multi", limit=opt_config["top_k"], - timeout=100 + timeout=120 ) return results @@ -421,7 +422,7 @@ def search_multi(multi_query: dict[str, str], user_config: dict[str, Any], opt_c query=query_embedding, # type: ignore[arg-type, unused-ignore] using="multi", limit=opt_config["top_k"], - timeout=100 + timeout=120 ) profilingLogger.info('end', extra={'activity': 'search', 'request_id': request_id}) return results From d618c0b1421f149e967d8e082a8161225e98a4c0 Mon Sep 17 00:00:00 2001 From: Hanna Reder Date: Thu, 11 Jun 2026 08:30:09 +0000 Subject: [PATCH 19/47] add resume mechanism for retrieval optimisation --- .../optimization/retrieval_optimization.py | 179 +++++++++++++++--- 1 file changed, 153 insertions(+), 26 deletions(-) diff --git a/learn2rag/optimization/retrieval_optimization.py b/learn2rag/optimization/retrieval_optimization.py index 57fd69c..d456a7c 100644 --- a/learn2rag/optimization/retrieval_optimization.py +++ b/learn2rag/optimization/retrieval_optimization.py @@ -1,6 +1,9 @@ """ RAG Retrieval Optimization. """ +import os +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"]="0" #Select GPU number 0 import argparse import json @@ -40,6 +43,80 @@ def load_registry(path: str = "registry.json") -> dict[str, Any]: with p.open() as f: return cast(dict[str, Any], json.load(f)) + +def _load_existing_trial_answers(answers_dir: pathlib.Path) -> list[dict[str, Any]]: + if not answers_dir.exists(): + return [] + trials: list[dict[str, Any]] = [] + for p in sorted(answers_dir.glob("trial_*_answers.json")): + try: + data = json.loads(p.read_text()) + if isinstance(data, dict) and "trial_id" in data: + trials.append(data) + except Exception as e: + logging.warning(f"Could not read {p}: {e}") + return trials + + +def _restore_state_from_existing(out: pathlib.Path, answers_dir: pathlib.Path) -> Dict[str, Any]: + state: Dict[str, Any] = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + + results_path = out / "optimization_results.json" + if results_path.exists(): + try: + results = json.loads(results_path.read_text()) + history = results.get("run_history", []) + convergence = results.get("convergence", []) + if isinstance(history, list) and history: + state["history"] = history + state["convergence"] = convergence if isinstance(convergence, list) else [] + state["trial_count"] = max(int(h.get("trial_id", 0)) for h in history) + state["best_cost"] = min(float(h.get("cost", 1.0)) for h in history) + return state + except Exception as e: + logging.warning(f"Could not read {results_path}: {e}") + + trials = _load_existing_trial_answers(answers_dir) + if not trials: + return state + + best_cost = 1.0 + history: list[dict[str, Any]] = [] + convergence: list[dict[str, Any]] = [] + for trial in sorted(trials, key=lambda t: int(t.get("trial_id", 0))): + tid = int(trial.get("trial_id", 0)) + cost = float(trial.get("cost", 1.0)) + best_cost = min(best_cost, cost) + history.append({ + "trial_id": tid, + "config": trial.get("config", {}), + "recall": float(trial.get("recall", 0.0)), + "avg_t_search": float(trial.get("avg_t_search", 0.0)), + "cost": cost, + "time_s": None, + "search_s": None, + "scoring_s": None, + }) + convergence.append({"trial": tid, "cost": cost, "best_cost": best_cost}) + + state["history"] = history + state["convergence"] = convergence + state["trial_count"] = max(int(t.get("trial_id", 0)) for t in trials) + state["best_cost"] = best_cost + return state + + +def _load_existing_importance(out: pathlib.Path) -> Dict[str, Any]: + path = out / "parameter_importance.json" + if not path.exists(): + return {} + try: + data = json.loads(path.read_text()) + return data if isinstance(data, dict) else {} + except Exception as e: + logging.warning(f"Could not read {path}: {e}") + return {} + def run_search(question: str, user_config: Dict[str, Any], working_config: Dict[str, Any]) -> Tuple[List[Any], float]: t0 = time.time() docs = asyncio.run(learn2rag.pipeline.search.search_authorized(question, user="anonymous", request_id=None, user_config=user_config, opt_config=working_config)) @@ -128,7 +205,7 @@ def objective(config: Configuration, logging.warning(f"Trial {tid}, q{q.get('id','?')} failed: {e}") predictions.append([]) goldens.append(q["ground_truth"]) - qa_pairs.append({**q, "generated_answer": "", "retrieved_context": ""}) + qa_pairs.append({**q, "retrieved_sources": ""}) if not predictions: return 1.0 @@ -139,8 +216,8 @@ def objective(config: Configuration, total_time = time.time() - t_start # objective function - w_recall = 1 - w_time = 0 + w_recall = 0.5 + w_time = 0.5 t_search_per_sample_upper = 50 max_time_s = t_search_per_sample_upper*len(predictions) time_cost = max(0.0, 1.0 - (t_search / max_time_s)) @@ -159,6 +236,12 @@ def objective(config: Configuration, "qa_pairs": qa_pairs } answers_file = answers_dir / f"trial_{tid}_answers.json" + while answers_file.exists(): + # Keep IDs monotonic when resuming from a partially persisted run. + tid += 1 + state["trial_count"] = tid + trial_answers["trial_id"] = tid + answers_file = answers_dir / f"trial_{tid}_answers.json" with open(answers_file, "w") as f: json.dump(trial_answers, f, indent=2, default=str) @@ -207,7 +290,15 @@ def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathli return result -def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[str, pathlib.Path], registry_path:str) -> Tuple[ +def run( + dataset_name: str, + max_questions: int, + n_trials: int, + output_dir: Union[str, pathlib.Path], + registry_path: str, + resume: bool = False, + n_trials_is_total: bool = True, +) -> Tuple[ Dict[str, Any], List[Any], Dict[str, Any]]: registry = load_registry(registry_path) datasets = registry["datasets"] @@ -289,27 +380,49 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ )) - scenario = Scenario( - cs, - deterministic=True, - n_trials=n_trials, - walltime_limit=172800, #7200, - seed=42, - output_directory=out / "smac_output", - ) - state: Dict[str, Any] = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + state: Dict[str, Any] + if resume: + state = _restore_state_from_existing(out, answers_dir) + else: + state = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + + already_done = int(state["trial_count"]) + remaining_trials = max(0, n_trials - already_done) if n_trials_is_total else n_trials + + best_cfg: Dict[str, Any] = {} + importance: Dict[str, Any] = {} + total_time = 0.0 + + if remaining_trials > 0: + scenario = Scenario( + cs, + deterministic=True, + n_trials=remaining_trials, + walltime_limit=172800, #7200, + seed=42, + output_directory=out / "smac_output", + ) + + smac = HyperparameterOptimizationFacade( + scenario=scenario, + target_function=lambda config, seed=0: objective(config, questions, state, answers_dir) + ) + t0 = time.time() + incumbent = smac.optimize() + if isinstance(incumbent, list): + incumbent = incumbent[0] + importance = param_importance(smac, out) + total_time = time.time() - t0 + best_cfg = incumbent.get_dictionary() + else: + logging.info("No remaining trials to run. Returning existing results.") + if state["history"]: + best_cfg = min(state["history"], key=lambda h: h["cost"]).get("config", {}) + importance = _load_existing_importance(out) + + if not best_cfg and state["history"]: + best_cfg = min(state["history"], key=lambda h: h["cost"]).get("config", {}) - smac = HyperparameterOptimizationFacade( - scenario=scenario, - target_function=lambda config, seed=0: objective(config, questions, state, answers_dir) - ) - t0 = time.time() - incumbent = smac.optimize() - if isinstance(incumbent, list): - incumbent = incumbent[0] - importance = param_importance(smac, out) - total_time = time.time() - t0 - best_cfg = incumbent.get_dictionary() best_trial = min( (h for h in state["history"] if h.get("config") == best_cfg), key=lambda h: h["cost"], @@ -340,6 +453,12 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ parser.add_argument("--logging-config", type=str) parser.add_argument("--registry", type=str, default="registry.json") parser.add_argument("--output_dir", type=str, default="optimization_results_baseline") + parser.add_argument("--resume", action="store_true") + parser.add_argument( + "--n_trials_is_total", + action="store_true", + help="Interpret --n_trials as the total desired trial count instead of additional trials.", + ) args, _ = parser.parse_known_args() final_output_dir = pathlib.Path(args.output_dir) @@ -348,13 +467,21 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ if not final_output_dir.exists() and env_out: final_output_dir = pathlib.Path(env_out).parent - incumbent, history, importance = run(args.dataset, args.max_questions, args.n_trials, final_output_dir, args.registry) + incumbent, history, importance = run( + args.dataset, + args.max_questions, + args.n_trials, + final_output_dir, + args.registry, + resume=args.resume, + n_trials_is_total=(args.n_trials_is_total or args.resume), + ) # incumbent, history, importance = run( # args.dataset, args.max_questions, args.n_trials, args.output_dir, # ) - best = min(history, key=lambda x: x["cost"]) + best = min(history, key=lambda x: x["cost"]) if history else None print(f"\nBest config: {dict(incumbent)}") # print(f"BERTScore (golden): {best['avg_bertscore_golden']:.4f}") if importance: From 5cf3c753b7e5fbedfa57be7a7a760c0011a6139d Mon Sep 17 00:00:00 2001 From: farshad68 Date: Thu, 11 Jun 2026 15:18:20 +0200 Subject: [PATCH 20/47] add type to function --- learn2rag/optimization/__init__.py | 4 ++- .../optimization/baseline_optimization.py | 36 +++++++++++-------- learn2rag/optimization/registry.json | 18 ++++++++++ learn2rag/pipeline/generate.py | 10 ++++-- .../compose/pipelines/optimization.yml | 2 ++ pyproject.toml | 6 +++- 6 files changed, 57 insertions(+), 19 deletions(-) create mode 100644 learn2rag/optimization/registry.json diff --git a/learn2rag/optimization/__init__.py b/learn2rag/optimization/__init__.py index 86410e8..8bad4b0 100644 --- a/learn2rag/optimization/__init__.py +++ b/learn2rag/optimization/__init__.py @@ -6,6 +6,8 @@ import copy import yaml import argparse +from typing import Any +from collections.abc import Mapping from . import baseline_optimization #TODO : now we need to copy the dataset to here manually it should consider in installation maybe ! @@ -66,7 +68,7 @@ def main() -> None: logging.info(f"Successfully updated opt_config at: {target_config_path}") -def deep_update(source, overrides): +def deep_update(source: dict[str, Any], overrides:Mapping[str, Any]) -> dict[str, Any]: """Recursively updates a dictionary.""" for key, value in overrides.items(): if isinstance(value, collections.abc.Mapping) and key in source: diff --git a/learn2rag/optimization/baseline_optimization.py b/learn2rag/optimization/baseline_optimization.py index be02a12..a2e9b60 100644 --- a/learn2rag/optimization/baseline_optimization.py +++ b/learn2rag/optimization/baseline_optimization.py @@ -9,10 +9,10 @@ import time import copy import os -from typing import Dict, Any, List, Union, Tuple - +from typing import Dict, Any, List, Union, Tuple, cast +from qdrant_client.models import ScoredPoint import numpy as np -from bert_score import score as bert_score +from bert_score import score as bert_score # type: ignore from ConfigSpace import ConfigurationSpace, Integer, Categorical, ForbiddenGreaterThanRelation, Configuration from smac import HyperparameterOptimizationFacade, Scenario @@ -21,12 +21,12 @@ import learn2rag.pipeline.search import learn2rag.pipeline.generate -def load_registry(path: str = "registry.json") -> dict: +def load_registry(path: str = "registry.json") -> dict[str, Any]: p = pathlib.Path(path) if not p.is_file(): logging.error("registry file not found") with p.open() as f: - return json.load(f) + return cast(Dict[str, Any], json.load(f)) def run_pipeline(question: str, user_config: Dict[str, Any], working_config: Dict[str, Any]) -> Tuple[str, str, float, float]: t0 = time.time() @@ -34,16 +34,19 @@ def run_pipeline(question: str, user_config: Dict[str, Any], working_config: Dic search_time = time.time() - t0 t0 = time.time() - answer = learn2rag.pipeline.generate.generate(question, docs, working_config) + answer = learn2rag.pipeline.generate.generate(question, docs.points, working_config) gen_time = time.time() - t0 doc_list = docs.points if hasattr(docs, "points") else docs context = "" if doc_list: - context = "\n\n".join([ - f"Source: {d.payload.get('path', 'unknown')}\nContent: {d.payload.get('content', '')}" - for d in doc_list - ]) + context_parts = [] + for d in doc_list: + payload = getattr(d, "payload", {}) or {} + path = payload.get("path", "unknown") if isinstance(payload, dict) else "unknown" + content = payload.get("content", "") if isinstance(payload, dict) else "" + context_parts.append(f"Source: {path}\nContent: {content}") + context = "\n\n".join(context_parts) return answer, context[:3000], search_time, gen_time #I removed seed because there are no use for it @@ -53,7 +56,7 @@ def objective(config: Configuration, dataset_name: str, state: Dict[str, Any], answers_dir: pathlib.Path - ,prompt_map + ,prompt_map: Dict[str, Any] ) -> float: state["trial_count"] += 1 tid = state["trial_count"] @@ -153,7 +156,7 @@ def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathli raw = {} for p in params: - groups = {} + groups : Dict[str, List[float]] = {} for c, cost in zip(configs, np.array(costs)): groups.setdefault(str(c[p]), []).append(cost) means = [np.mean(g) for g in groups.values()] @@ -161,7 +164,7 @@ def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathli total = sum(raw.values()) imp = {p: round(v / total, 4) for p, v in raw.items()} if total > 0 else raw - ranking = sorted(imp, key=imp.get, reverse=True) + ranking = sorted(imp, key=lambda k: imp[k], reverse=True) result = {"method": "variance_based", "ranking": ranking, "individual": imp} with open(output_path / "parameter_importance.json", "w") as f: json.dump(result, f, indent=2) @@ -170,6 +173,7 @@ def param_importance(smac: HyperparameterOptimizationFacade, output_path: pathli def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[str, pathlib.Path],registry_path:str) -> Tuple[ Dict[str, Any], List[Any], Dict[str, Any]]: + logging.info(f"registry_path is : {registry_path}") registry = load_registry(registry_path) datasets = registry["datasets"] if dataset_name not in datasets: @@ -220,9 +224,13 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ ) t0 = time.time() incumbent = smac.optimize() + if isinstance(incumbent, list): + best_cfg = incumbent[0].get_dictionary() + else: + best_cfg = incumbent.get_dictionary() importance = param_importance(smac, out) total_time = time.time() - t0 - best_cfg = incumbent.get_dictionary() + #best_cfg = incumbent.get_dictionary() results_path = out / "optimization_results.json" results_path.write_text(json.dumps({ "best_config": best_cfg, diff --git a/learn2rag/optimization/registry.json b/learn2rag/optimization/registry.json new file mode 100644 index 0000000..941c261 --- /dev/null +++ b/learn2rag/optimization/registry.json @@ -0,0 +1,18 @@ +{ + "datasets": { + "WikiEval": { + "subdirectory": "", "split": "train", + "fields": {"q": "question", "a": "answer", "id": "id"} + } + }, + "prompts": { + "default": + "# Role and Objective\nYou will act as a smart AI chatbot that answers questions only by using the content from the provided information list.\n\n # Instructions\n- Respond in the language of the question.\n - Answer clear and concise.\n- Only use the provided information.\n - NEVER use your general knowledge.\n\n # Information:\n{context}" + , + "concise": + "Answer the question using ONLY the provided information. Be concise and direct. If the information does not contain the answer, say so.\n\n Information:\n{context}" + , + "detailed": + "You are a knowledgeable assistant. Using ONLY the provided information below, answer the question thoroughly. Cite your sources. If the information is insufficient, state that clearly.\n\n Information:\n{context}" + } +} \ No newline at end of file diff --git a/learn2rag/pipeline/generate.py b/learn2rag/pipeline/generate.py index 64e3efc..c088ef5 100644 --- a/learn2rag/pipeline/generate.py +++ b/learn2rag/pipeline/generate.py @@ -1,6 +1,7 @@ -from typing import Any, Generator +from typing import Any, Generator, Sequence import logging from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate, ChatPromptTemplate + from qdrant_client.http.models import ScoredPoint from .llm import llm @@ -14,11 +15,14 @@ {content} """ -def generate(query: str, search_results: list[ScoredPoint], opt_config: dict[str, Any]) -> Any: +def generate(query: str, search_results: Sequence[ScoredPoint], opt_config: dict[str, Any]) -> Any: assert llm is not None if hasattr(search_results, "points"): search_results = search_results.points - context = "\n\n".join([context_template.format(source=result.payload['path'], content=result.payload['content']) for result in search_results]) # type: ignore[index] + context = "\n\n".join([ + context_template.format(source=result.payload['path'], content=result.payload['content']) + for result in search_results if result.payload + ]) system_message = SystemMessagePromptTemplate.from_template(opt_config["prompt"]) user_message = HumanMessagePromptTemplate.from_template("{question}") prompt = ChatPromptTemplate.from_messages([system_message, user_message]) diff --git a/learn2rag/ui/templates/compose/pipelines/optimization.yml b/learn2rag/ui/templates/compose/pipelines/optimization.yml index c721f88..c368649 100644 --- a/learn2rag/ui/templates/compose/pipelines/optimization.yml +++ b/learn2rag/ui/templates/compose/pipelines/optimization.yml @@ -99,6 +99,8 @@ services: - 'learn2rag.optimization' - '--logging-config' - '{{storage_path}}/logging_config.yml' + - '--registry-path' + - '{{learn2rag_path}}/learn2rag/optimization/registry.json' environment: LEARN2RAG_PATH: '{{learn2rag_path}}' QDRANT__SERVICE__HTTP_PORT: '{{ports.qdrant_http}}' diff --git a/pyproject.toml b/pyproject.toml index e53ed41..a0adc3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,7 +208,11 @@ build-backend = "setuptools.build_meta" requires = ["setuptools >= 77.0.3"] [tool.mypy] files = "learn2rag" -exclude = "^learn2rag/pipeline/scripts/" +exclude = ["^learn2rag/pipeline/scripts/", + "^learn2rag/optimization/contrastive_optimization.py", +"generate_contrastive_answers.py", +"self_judge_optimization.py", +"tool_read_data_set.py"] strict = true [[tool.mypy.overrides]] module = [ From 3bab6bcaf77de41d0d8cd6a8811650242d72a269 Mon Sep 17 00:00:00 2001 From: farshad68 Date: Fri, 12 Jun 2026 11:29:48 +0200 Subject: [PATCH 21/47] set api_key with mock value for test --- learn2rag/tests/test_learn2rag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learn2rag/tests/test_learn2rag.py b/learn2rag/tests/test_learn2rag.py index 095de51..d51719d 100644 --- a/learn2rag/tests/test_learn2rag.py +++ b/learn2rag/tests/test_learn2rag.py @@ -24,7 +24,7 @@ def setUp(self) -> None: self.storage_path = Path(save_data_path('Learn2RAG', 'tests')) self.storage_path.mkdir(parents=True, exist_ok=True) self.openai_client = OpenAI( - api_key='', + api_key='mock-key', base_url=f'http://localhost:{self.rag_port}', max_retries=0, ) From 478671505b4efb4785b28a892e855cb3247a8c90 Mon Sep 17 00:00:00 2001 From: farshad68 Date: Fri, 12 Jun 2026 14:12:39 +0200 Subject: [PATCH 22/47] remove chunk_size and chunk_overlap because optimizing them with out reindex is not functional --- learn2rag/optimization/baseline_optimization.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/learn2rag/optimization/baseline_optimization.py b/learn2rag/optimization/baseline_optimization.py index a2e9b60..ac0b669 100644 --- a/learn2rag/optimization/baseline_optimization.py +++ b/learn2rag/optimization/baseline_optimization.py @@ -66,8 +66,8 @@ def objective(config: Configuration, working_cfg = copy.deepcopy(opt_config) working_cfg.update({ "top_k": cfg["top_k"], - "chunk_size": cfg["chunk_size"], - "chunk_overlap": cfg["chunk_overlap"], + # "chunk_size": cfg["chunk_size"], + # "chunk_overlap": cfg["chunk_overlap"], "prompt": prompt_map[cfg["prompt_template"]], }) @@ -203,11 +203,11 @@ def run(dataset_name: str, max_questions: int, n_trials: int, output_dir: Union[ cs = ConfigurationSpace(seed=42) cs.add([ Integer("top_k", (1, 20), default=4), - Integer("chunk_size", (200, 4000), default=2000), - Integer("chunk_overlap", (0, 500), default=200), + #Integer("chunk_size", (200, 4000), default=2000), + #Integer("chunk_overlap", (0, 500), default=200), Categorical("prompt_template", list(prompt_map.keys()), default="default"), ]) - cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) + #cs.add(ForbiddenGreaterThanRelation(cs["chunk_overlap"], cs["chunk_size"])) scenario = Scenario( cs, deterministic=True, From 8a351dec4a020c2771242d109f9ae1237aa9a11c Mon Sep 17 00:00:00 2001 From: Hanna Reder Date: Fri, 12 Jun 2026 15:01:53 +0000 Subject: [PATCH 23/47] fix Ollama timeout and colbert reranking issues --- .../optimization/retrieval_optimization.py | 215 +++++++++++++++++- learn2rag/pipeline/llm.py | 21 ++ learn2rag/pipeline/rewrite.py | 99 ++++++-- learn2rag/pipeline/search.py | 44 +++- 4 files changed, 347 insertions(+), 32 deletions(-) diff --git a/learn2rag/optimization/retrieval_optimization.py b/learn2rag/optimization/retrieval_optimization.py index d456a7c..f7f1b13 100644 --- a/learn2rag/optimization/retrieval_optimization.py +++ b/learn2rag/optimization/retrieval_optimization.py @@ -6,6 +6,7 @@ os.environ["CUDA_VISIBLE_DEVICES"]="0" #Select GPU number 0 import argparse +import datetime import json import logging import pathlib @@ -13,6 +14,8 @@ import copy import os import asyncio +import subprocess +import sys from typing import Dict, Any, List, Union, Tuple, cast import numpy as np @@ -61,6 +64,13 @@ def _load_existing_trial_answers(answers_dir: pathlib.Path) -> list[dict[str, An def _restore_state_from_existing(out: pathlib.Path, answers_dir: pathlib.Path) -> Dict[str, Any]: state: Dict[str, Any] = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} + trial_answers = _load_existing_trial_answers(answers_dir) + answers_by_id = { + int(t.get("trial_id", 0)): t + for t in trial_answers + if isinstance(t, dict) and t.get("trial_id") is not None + } + results_path = out / "optimization_results.json" if results_path.exists(): try: @@ -68,15 +78,45 @@ def _restore_state_from_existing(out: pathlib.Path, answers_dir: pathlib.Path) - history = results.get("run_history", []) convergence = results.get("convergence", []) if isinstance(history, list) and history: - state["history"] = history - state["convergence"] = convergence if isinstance(convergence, list) else [] - state["trial_count"] = max(int(h.get("trial_id", 0)) for h in history) - state["best_cost"] = min(float(h.get("cost", 1.0)) for h in history) + merged_history: dict[int, dict[str, Any]] = { + int(h.get("trial_id", 0)): dict(h) + for h in history + if isinstance(h, dict) and h.get("trial_id") is not None + } + + for tid, trial in answers_by_id.items(): + previous = merged_history.get(tid, {}) + merged_history[tid] = { + "trial_id": tid, + "config": trial.get("config", previous.get("config", {})), + "recall": float(trial.get("recall") or previous.get("recall") or 0.0), + "avg_t_search": float(trial.get("avg_t_search") or previous.get("avg_t_search") or 0.0), + "cost": float(trial.get("cost") or previous.get("cost") or 1.0), + "time_s": previous.get("time_s"), + "search_s": previous.get("search_s"), + "scoring_s": previous.get("scoring_s"), + } + + merged_list = [merged_history[tid] for tid in sorted(merged_history)] + best_cost = 1.0 + rebuilt_convergence: list[dict[str, Any]] = [] + for entry in merged_list: + best_cost = min(best_cost, float(entry.get("cost", 1.0))) + rebuilt_convergence.append({ + "trial": int(entry.get("trial_id", 0)), + "cost": float(entry.get("cost", 1.0)), + "best_cost": best_cost, + }) + + state["history"] = merged_list + state["convergence"] = rebuilt_convergence if not isinstance(convergence, list) or len(rebuilt_convergence) != len(convergence) else convergence + state["trial_count"] = max(int(h.get("trial_id", 0)) for h in merged_list) + state["best_cost"] = best_cost return state except Exception as e: logging.warning(f"Could not read {results_path}: {e}") - trials = _load_existing_trial_answers(answers_dir) + trials = trial_answers if not trials: return state @@ -117,7 +157,155 @@ def _load_existing_importance(out: pathlib.Path) -> Dict[str, Any]: logging.warning(f"Could not read {path}: {e}") return {} + +def _find_latest_optimization_file(smac_output_dir: pathlib.Path) -> Union[pathlib.Path, None]: + if not smac_output_dir.exists(): + return None + candidates = list(smac_output_dir.rglob("optimization.json")) + if not candidates: + return None + return max(candidates, key=lambda p: p.stat().st_mtime) + + +def _parse_last_update(value: Any) -> Union[float, None]: + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + # Handle ISO values like 2026-06-11T12:34:56.123456+00:00 or trailing Z. + dt = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return dt.timestamp() + except ValueError: + return None + return None + + +def _last_update_age_seconds(smac_output_dir: pathlib.Path) -> Union[float, None]: + optimization_file = _find_latest_optimization_file(smac_output_dir) + if optimization_file is None: + return None + try: + data = json.loads(optimization_file.read_text()) + except Exception as e: + logging.warning(f"Could not read {optimization_file}: {e}") + return None + ts = _parse_last_update(data.get("last_update")) if isinstance(data, dict) else None + if ts is None: + return None + return max(0.0, time.time() - ts) + + +def _run_search_heartbeat_age_seconds(heartbeat_file: pathlib.Path) -> Union[float, None]: + if not heartbeat_file.exists(): + return None + try: + return max(0.0, time.time() - heartbeat_file.stat().st_mtime) + except OSError as e: + logging.warning(f"Could not stat {heartbeat_file}: {e}") + return None + + +def _touch_run_search_heartbeat() -> None: + heartbeat_path = os.environ.get("L2R_RUN_SEARCH_HEARTBEAT_FILE") + if not heartbeat_path: + return + try: + p = pathlib.Path(heartbeat_path) + p.parent.mkdir(parents=True, exist_ok=True) + p.touch() + except Exception as e: + logging.warning(f"Could not update run_search heartbeat at {heartbeat_path}: {e}") + + +def _build_worker_command(args: argparse.Namespace, final_output_dir: pathlib.Path) -> List[str]: + cmd = [ + sys.executable, + "-m", + "learn2rag.optimization.retrieval_optimization", + "--dataset", args.dataset, + "--max_questions", str(args.max_questions), + "--n_trials", str(args.n_trials), + "--registry", args.registry, + "--output_dir", str(final_output_dir), + "--resume", + ] + if args.n_trials_is_total or args.resume: + cmd.append("--n_trials_is_total") + if args.logging_config: + cmd.extend(["--logging-config", args.logging_config]) + return cmd + + +def run_with_watchdog(args: argparse.Namespace, final_output_dir: pathlib.Path) -> int: + stale_after_s = max(1, args.watchdog_stale_minutes * 60) + run_search_stale_after_s = max(1, args.watchdog_run_search_stale_minutes * 60) + restart_wait_s = max(1, args.watchdog_restart_delay_minutes * 60) + poll_s = max(5, args.watchdog_poll_seconds) + + dataset_out = final_output_dir / args.dataset + smac_output_dir = dataset_out / "smac_output" + run_search_heartbeat_file = dataset_out / "run_search_heartbeat.txt" + worker_cmd = _build_worker_command(args, final_output_dir) + + restart_count = 0 + while True: + logging.info(f"Starting optimization worker (restart #{restart_count})") + try: + run_search_heartbeat_file.unlink(missing_ok=True) + except OSError as e: + logging.warning(f"Could not reset heartbeat file {run_search_heartbeat_file}: {e}") + + worker_env = os.environ.copy() + worker_env["L2R_RUN_SEARCH_HEARTBEAT_FILE"] = str(run_search_heartbeat_file) + proc = subprocess.Popen(worker_cmd, env=worker_env) + stale_detected = False + + while proc.poll() is None: + time.sleep(poll_s) + age = _last_update_age_seconds(smac_output_dir) + if age is not None and age > stale_after_s: + stale_detected = True + logging.warning( + f"Detected stale optimization.json update (age={age:.0f}s > {stale_after_s}s). " + "Terminating worker for restart." + ) + proc.terminate() + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + break + + run_search_age = _run_search_heartbeat_age_seconds(run_search_heartbeat_file) + if run_search_age is not None and run_search_age > run_search_stale_after_s: + stale_detected = True + logging.warning( + f"Detected stale run_search heartbeat (age={run_search_age:.0f}s > {run_search_stale_after_s}s). " + "Terminating worker for restart." + ) + proc.terminate() + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + break + + if not stale_detected: + return_code = proc.returncode if proc.returncode is not None else 1 + if return_code == 0: + logging.info("Optimization worker finished successfully.") + else: + logging.error(f"Optimization worker exited with return code {return_code}.") + return return_code + + restart_count += 1 + logging.info(f"Sleeping {restart_wait_s}s before resuming optimization.") + time.sleep(restart_wait_s) + def run_search(question: str, user_config: Dict[str, Any], working_config: Dict[str, Any]) -> Tuple[List[Any], float]: + _touch_run_search_heartbeat() t0 = time.time() docs = asyncio.run(learn2rag.pipeline.search.search_authorized(question, user="anonymous", request_id=None, user_config=user_config, opt_config=working_config)) search_time = time.time() - t0 @@ -387,7 +575,8 @@ def run( state = {"trial_count": 0, "best_cost": 1.0, "convergence": [], "history": []} already_done = int(state["trial_count"]) - remaining_trials = max(0, n_trials - already_done) if n_trials_is_total else n_trials + target_trials = max(already_done, n_trials) if n_trials_is_total else already_done + n_trials + remaining_trials = max(0, target_trials - already_done) best_cfg: Dict[str, Any] = {} importance: Dict[str, Any] = {} @@ -397,7 +586,7 @@ def run( scenario = Scenario( cs, deterministic=True, - n_trials=remaining_trials, + n_trials=target_trials, walltime_limit=172800, #7200, seed=42, output_directory=out / "smac_output", @@ -407,13 +596,14 @@ def run( scenario=scenario, target_function=lambda config, seed=0: objective(config, questions, state, answers_dir) ) + t0 = time.time() incumbent = smac.optimize() if isinstance(incumbent, list): incumbent = incumbent[0] importance = param_importance(smac, out) total_time = time.time() - t0 - best_cfg = incumbent.get_dictionary() + best_cfg = dict(incumbent) else: logging.info("No remaining trials to run. Returning existing results.") if state["history"]: @@ -459,6 +649,11 @@ def run( action="store_true", help="Interpret --n_trials as the total desired trial count instead of additional trials.", ) + parser.add_argument("--watchdog", action="store_true", help="Restart optimization if SMAC last_update is stale.") + parser.add_argument("--watchdog_stale_minutes", type=int, default=90) + parser.add_argument("--watchdog_run_search_stale_minutes", type=int, default=7) + parser.add_argument("--watchdog_restart_delay_minutes", type=int, default=5) + parser.add_argument("--watchdog_poll_seconds", type=int, default=60) args, _ = parser.parse_known_args() final_output_dir = pathlib.Path(args.output_dir) @@ -467,6 +662,10 @@ def run( if not final_output_dir.exists() and env_out: final_output_dir = pathlib.Path(env_out).parent + if args.watchdog: + exit_code = run_with_watchdog(args, final_output_dir) + raise SystemExit(exit_code) + incumbent, history, importance = run( args.dataset, args.max_questions, diff --git a/learn2rag/pipeline/llm.py b/learn2rag/pipeline/llm.py index 18cc061..5f0db5f 100644 --- a/learn2rag/pipeline/llm.py +++ b/learn2rag/pipeline/llm.py @@ -1,5 +1,6 @@ import logging import os +import httpx from pydantic import SecretStr from langchain_core.language_models.chat_models import BaseChatModel from langchain_ollama import ChatOllama @@ -9,6 +10,25 @@ logger = logging.getLogger(__name__) +def _env_float(name: str, default: float) -> float: + value = os.environ.get(name) + if value is None: + return default + try: + return float(value) + except ValueError: + logger.warning("Invalid float for %s=%r. Falling back to %s.", name, value, default) + return default + + +def _ollama_timeout() -> httpx.Timeout: + timeout_s = max(1.0, _env_float("L2R_OLLAMA_TIMEOUT_SECONDS", 90.0)) + connect_s = min(10.0, timeout_s) + write_s = min(30.0, timeout_s) + pool_s = min(10.0, timeout_s) + return httpx.Timeout(timeout=timeout_s, connect=connect_s, read=timeout_s, write=write_s, pool=pool_s) + + class LLMClient(): # ID is used as a key to store in user data, should not be changed ID: str @@ -50,6 +70,7 @@ def __init__(self, *, url: str, token: str | None, model: str, proxy: str | None client_kwargs={ 'headers': {'Authorization': f'Bearer {token}'} if token else {}, 'proxy': proxy, + 'timeout': _ollama_timeout(), }, ) diff --git a/learn2rag/pipeline/rewrite.py b/learn2rag/pipeline/rewrite.py index 42e405e..e2eb87f 100644 --- a/learn2rag/pipeline/rewrite.py +++ b/learn2rag/pipeline/rewrite.py @@ -1,8 +1,73 @@ from langchain_core.messages import SystemMessage, HumanMessage import ast +import logging +import os +import time from .llm import llm +logger = logging.getLogger(__name__) + + +def _env_int(name: str, default: int) -> int: + value = os.environ.get(name) + if value is None: + return default + try: + return int(value) + except ValueError: + logger.warning("Invalid int for %s=%r. Falling back to %s.", name, value, default) + return default + + +def _env_float(name: str, default: float) -> float: + value = os.environ.get(name) + if value is None: + return default + try: + return float(value) + except ValueError: + logger.warning("Invalid float for %s=%r. Falling back to %s.", name, value, default) + return default + + +def _invoke_llm(messages: list[SystemMessage | HumanMessage], *, purpose: str) -> str: + if llm is None: + return '' + + max_attempts = max(1, _env_int("L2R_OLLAMA_MAX_RETRIES", 2)) + retry_sleep_s = max(0.0, _env_float("L2R_OLLAMA_RETRY_BACKOFF_SECONDS", 2.0)) + + for attempt in range(1, max_attempts + 1): + try: + t0 = time.time() + logger.info("llm_invoke_start purpose=%s attempt=%d/%d", purpose, attempt, max_attempts) + response = llm.invoke(messages, stream=False) + duration_s = time.time() - t0 + logger.info( + "llm_invoke_done purpose=%s attempt=%d/%d duration_s=%.2f", + purpose, + attempt, + max_attempts, + duration_s, + ) + content = response.content + return content.strip() if isinstance(content, str) else '' + except Exception as exc: + logger.warning( + "llm_invoke_failed purpose=%s attempt=%d/%d error=%s", + purpose, + attempt, + max_attempts, + exc, + ) + if attempt < max_attempts and retry_sleep_s > 0: + time.sleep(retry_sleep_s * attempt) + + logger.error("llm_invoke_give_up purpose=%s attempts=%d", purpose, max_attempts) + return '' + + # future todo: add history handling / add state handling for loops in pipeline try: @@ -25,15 +90,11 @@ def rewrite_query(user_query: str) -> str: - Return only the rewritten query text. """ - response = llm.invoke([ + content = _invoke_llm([ SystemMessage(content=system_message_rewrite_query), HumanMessage(content=user_query), - ]) - - content = response.content - if isinstance(content, str): - return content.strip() - return '' + ], purpose="rewrite_query") + return content def generate_subqueries(user_query: str, n: int=3) -> list[str]: @@ -53,21 +114,19 @@ def generate_subqueries(user_query: str, n: int=3) -> list[str]: {synonym_list} """ - response = llm.invoke([ + content = _invoke_llm([ SystemMessage(content=system_message_generate_subqueries), HumanMessage(content=user_query), - ]) - - content = response.content - if not isinstance(content, str): + ], purpose="generate_subqueries") + if not content: return [] try: result = ast.literal_eval(content.strip()) if isinstance(result, list): return [str(x).strip() for x in result if str(x).strip()] - except Exception: - pass + except Exception as exc: + logger.warning("generate_subqueries_parse_failed query=%r error=%s content=%r", user_query, exc, content[:500]) return [] @@ -91,20 +150,18 @@ def generate_keywords(user_query: str, n: int=3) -> list[str]: {synonym_list} """ - response = llm.invoke([ + content = _invoke_llm([ SystemMessage(content=system_message_generate_keywords), HumanMessage(content=user_query), - ]) - - content = response.content - if not isinstance(content, str): + ], purpose="generate_keywords") + if not content: return [] try: result = ast.literal_eval(content.strip()) if isinstance(result, list): return [str(x).strip() for x in result if str(x).strip()] - except Exception: - pass + except Exception as exc: + logger.warning("generate_keywords_parse_failed query=%r error=%s content=%r", user_query, exc, content[:500]) return [] diff --git a/learn2rag/pipeline/search.py b/learn2rag/pipeline/search.py index 617eba7..bece13b 100644 --- a/learn2rag/pipeline/search.py +++ b/learn2rag/pipeline/search.py @@ -119,6 +119,7 @@ def _rerank_points_with_colbert( *, top_k: int, opt_config: dict[str, Any], + user_config: dict[str, Any], ) -> list[ScoredPoint]: collection_name = user_config["collection_name"] qdrant = Qdrant(collection_name=collection_name, opt_config=opt_config) @@ -197,8 +198,26 @@ def _collect_query_points( extra={'activity': '_collect_query_points', 'request_id': request_id}, ) - for sq in subqueries: + for idx, sq in enumerate(subqueries, start=1): + profilingLogger.info( + "subquery_search_start query=%r subquery_index=%d/%d subquery=%r top_k=%s", + query, + idx, + len(subqueries), + sq, + opt_config_subqueries["top_k"], + extra={'activity': '_collect_query_points', 'request_id': request_id}, + ) sq_results = search(sq, user_config, opt_config_subqueries, request_id=request_id) + profilingLogger.info( + "subquery_search_done query=%r subquery_index=%d/%d subquery=%r points=%d", + query, + idx, + len(subqueries), + sq, + len(sq_results.points), + extra={'activity': '_collect_query_points', 'request_id': request_id}, + ) points_all.extend(sq_results.points) if rewrite_mode in ["keywords", "subqueries_keywords"]: @@ -217,8 +236,26 @@ def _collect_query_points( extra={'activity': '_collect_query_points', 'request_id': request_id}, ) - for kw in keywords: + for idx, kw in enumerate(keywords, start=1): + profilingLogger.info( + "keyword_search_start query=%r keyword_index=%d/%d keyword=%r top_k=%s", + query, + idx, + len(keywords), + kw, + opt_config_keywords["top_k"], + extra={'activity': '_collect_query_points', 'request_id': request_id}, + ) kw_results = search(kw, user_config, opt_config_keywords, request_id=request_id) + profilingLogger.info( + "keyword_search_done query=%r keyword_index=%d/%d keyword=%r points=%d", + query, + idx, + len(keywords), + kw, + len(kw_results.points), + extra={'activity': '_collect_query_points', 'request_id': request_id}, + ) points_all.extend(kw_results.points) points = _sort_and_deduplicate(points_all) @@ -251,7 +288,8 @@ def _collect_query_points( query, points, top_k=opt_config["top_k_reranker"], - opt_config=opt_config + opt_config=opt_config, + user_config=user_config, ) else: points = points[:opt_config["top_k"]] From bd9bedb947763c4c981eaa7e7e30a2e7ec43e7d2 Mon Sep 17 00:00:00 2001 From: denkv Date: Tue, 16 Jun 2026 14:21:28 +0200 Subject: [PATCH 24/47] Add an ability to upload traning examples file --- learn2rag/ui/__init__.py | 34 +++++++++++++++ .../ui/templates/pipelines_details_page.html | 43 +++++++++++++++++++ learn2rag/ui/templates/pipelines_list.html | 6 +++ 3 files changed, 83 insertions(+) create mode 100644 learn2rag/ui/templates/pipelines_details_page.html diff --git a/learn2rag/ui/__init__.py b/learn2rag/ui/__init__.py index cb37f4f..e763872 100644 --- a/learn2rag/ui/__init__.py +++ b/learn2rag/ui/__init__.py @@ -12,6 +12,7 @@ import time from typing import Any import urllib +from itertools import islice from babel import negotiate_locale from flask import Flask, flash, redirect as flask_redirect, render_template, request, make_response, url_for @@ -479,6 +480,39 @@ def start_pipeline(name: str, pipeline: dict[str, Any], template_name: str) -> N # TODO "load" the corresponding Ollama model + @app.get('/pipelines/') + def pipeline_details(name: str) -> 'str | werkzeug.wrappers.response.Response': + pipeline = learn2rag.data.get_entry(app.instance_path, 'pipelines', name) + if pipeline is None: + flash(pgettext('flash', 'The requested pipeline is not found'), 'error') + return redirect(url_for('pipelines_list')) + storage_path = Path(pipeline['storage_path']) + try: + with (storage_path / 'training.csv').open() as training_file: + training_examples = ''.join(islice(training_file, 3)) + except FileNotFoundError: + training_examples = None + return render_template( + 'pipelines_details_page.html', + name=name, + pipeline=pipeline, + training_examples=training_examples, + ) + + @app.post('/pipelines//training') + def pipeline_details_training_data(name: str) -> 'str | werkzeug.wrappers.response.Response': + pipeline = learn2rag.data.get_entry(app.instance_path, 'pipelines', name) + if pipeline is None: + flash(pgettext('flash', 'The requested pipeline is not found'), 'error') + return redirect(url_for('pipelines_list')) + try: + training_file = request.files['trainingFile'] + training_file.save(Path(pipeline['storage_path']) / 'training.csv') + except Exception as e: + app.logger.exception(e) + flash(pgettext('flash', 'Could not save the file'), 'error') + return redirect(url_for('pipeline_details', name=name)) + @app.post('/pipelines/') def pipeline_action(name: str) -> 'str | werkzeug.wrappers.response.Response': pipeline = learn2rag.data.get_entry(app.instance_path, 'pipelines', name) diff --git a/learn2rag/ui/templates/pipelines_details_page.html b/learn2rag/ui/templates/pipelines_details_page.html new file mode 100644 index 0000000..5916ba8 --- /dev/null +++ b/learn2rag/ui/templates/pipelines_details_page.html @@ -0,0 +1,43 @@ +{% extends 'base.html' %} + +{% block header %} +

{% block title %}{{pipeline.label}}{% endblock %}

+{% endblock %} + +{% block content %} +

{{pgettext('header', 'Details')}}

+{{pipeline.label}} +
+

{{pgettext('header', 'Optimization')}}

+{% if training_examples %} +
+ + + + + + + + + + +
{{pgettext('header', 'Training data')}}
+
{{training_examples}}…
+
+
+{% endif %} +
+ +
+ +
+
+ +
+
+{% endblock %} diff --git a/learn2rag/ui/templates/pipelines_list.html b/learn2rag/ui/templates/pipelines_list.html index 8b5100e..f636f4a 100644 --- a/learn2rag/ui/templates/pipelines_list.html +++ b/learn2rag/ui/templates/pipelines_list.html @@ -83,6 +83,12 @@ {{pgettext('button', 'More')}}