1use indexmap::IndexMap;
8use sokuto_core::contract::model_spec::MAX_NUM_OPTIONS;
9use sokuto_core::schema::{Question, QuestionType};
10
11use crate::engine::InferenceEngine;
12use crate::error::{Result, RuntimeError};
13use crate::tokenizer::JevTokenizer;
14
15pub const DEFAULT_COARSE_THRESHOLD: usize = 30;
17
18pub const DEFAULT_TOP_M_CANDIDATES: usize = 20;
20
21pub const DEFAULT_NEGATIVE_KEYS: &[&str] = &[
23 "none",
24 "other",
25 "others",
26 "none_of_the_above",
27 "out_of_scope",
28 "unknown",
29 "not_applicable",
30 "na",
31 "n/a",
32 "該当なし",
33 "その他",
34 "不明",
35 "対象外",
36];
37
38#[derive(Debug, Clone, PartialEq, Eq)]
40pub struct CoarseToFineConfig {
41 pub threshold: usize,
43 pub top_m: usize,
45 pub preserve_negative: bool,
47 pub negative_keys: Vec<String>,
49}
50
51impl Default for CoarseToFineConfig {
52 fn default() -> Self {
53 Self {
54 threshold: DEFAULT_COARSE_THRESHOLD,
55 top_m: DEFAULT_TOP_M_CANDIDATES,
56 preserve_negative: true,
57 negative_keys: DEFAULT_NEGATIVE_KEYS
58 .iter()
59 .map(|&s| s.to_string())
60 .collect(),
61 }
62 }
63}
64
65impl CoarseToFineConfig {
66 pub fn should_trigger(&self, candidate_count: usize) -> bool {
71 candidate_count > self.threshold
72 }
73
74 pub fn validate(&self) -> Result<()> {
76 if self.top_m == 0 {
77 return Err(RuntimeError::InvalidQuestion(
78 "top_m は 1 以上である必要があります。".to_string(),
79 ));
80 }
81 if self.top_m > MAX_NUM_OPTIONS {
82 return Err(RuntimeError::InvalidQuestion(format!(
83 "top_m ({}) はモデル上限 ({}) を超えることはできません。",
84 self.top_m, MAX_NUM_OPTIONS
85 )));
86 }
87 Ok(())
88 }
89}
90
91pub fn is_negative_candidate(key: &str, description: &str, negative_keys: &[String]) -> bool {
100 let lower_key = key.trim().to_lowercase();
101 if negative_keys
102 .iter()
103 .any(|nk| nk.to_lowercase() == lower_key)
104 {
105 return true;
106 }
107
108 let lower_desc = description.trim().to_lowercase();
109 let negative_desc_phrases = [
110 "none of the above",
111 "out of scope",
112 "not applicable",
113 "other inquiry",
114 "general inquiry",
115 "上記のいずれにも該当しない",
116 "該当なし",
117 "その他のお問い合わせ",
118 "サポート外",
119 ];
120
121 negative_desc_phrases
122 .iter()
123 .any(|phrase| lower_desc.contains(phrase))
124}
125
126#[derive(Debug, Clone, Default)]
131pub struct CandidateEmbeddingCache {
132 embeddings: IndexMap<String, Vec<f32>>,
134 dimension: usize,
136}
137
138impl CandidateEmbeddingCache {
139 pub fn new() -> Self {
141 Self::default()
142 }
143
144 pub fn insert(&mut self, key: impl Into<String>, mut vector: Vec<f32>) -> Result<()> {
150 let key_str = key.into();
151 if vector.is_empty() {
152 return Err(RuntimeError::InvalidTensorData(format!(
153 "キー '{key_str}' の埋め込みベクトルが空です。"
154 )));
155 }
156
157 if self.dimension == 0 {
158 self.dimension = vector.len();
159 } else if vector.len() != self.dimension {
160 return Err(RuntimeError::InvalidTensorData(format!(
161 "埋め込み次元数不一致: 期待値={}, 実際={} (キー='{key_str}')。",
162 self.dimension,
163 vector.len()
164 )));
165 }
166
167 let norm_sq: f32 = vector.iter().map(|v| v * v).sum();
169 let norm = norm_sq.sqrt();
170 if norm > 1e-12 {
171 let inv_norm = 1.0 / norm;
172 for v in &mut vector {
173 *v *= inv_norm;
174 }
175 }
176
177 self.embeddings.insert(key_str, vector);
178 Ok(())
179 }
180
181 pub fn get(&self, key: &str) -> Option<&[f32]> {
183 self.embeddings.get(key).map(|v| v.as_slice())
184 }
185
186 pub fn dimension(&self) -> usize {
188 self.dimension
189 }
190
191 pub fn len(&self) -> usize {
193 self.embeddings.len()
194 }
195
196 pub fn is_empty(&self) -> bool {
198 self.embeddings.is_empty()
199 }
200
201 pub fn cosine_similarity(v1: &[f32], v2: &[f32]) -> f32 {
203 if v1.len() != v2.len() || v1.is_empty() {
204 return 0.0;
205 }
206 v1.iter().zip(v2.iter()).map(|(a, b)| a * b).sum()
207 }
208}
209
210pub trait CoarseScorer {
212 fn score_candidates(
221 &self,
222 query: &str,
223 candidates: &IndexMap<String, String>,
224 ) -> Result<Vec<f32>>;
225}
226
227pub trait HierarchicalCoarseScorer {
232 fn score_coarse(
242 &self,
243 state: &str,
244 instructions: &str,
245 coarse_categories: &IndexMap<String, String>,
246 ) -> Result<IndexMap<String, f64>>;
247}
248
249pub struct ModelDrivenCoarseScorer<'a> {
254 engine: &'a InferenceEngine,
255 tokenizer: &'a JevTokenizer,
256 calib_config: Option<&'a sokuto_core::contract::calibration::CalibrationConfig>,
257}
258
259impl<'a> ModelDrivenCoarseScorer<'a> {
260 pub fn new(engine: &'a InferenceEngine, tokenizer: &'a JevTokenizer) -> Self {
262 Self {
263 engine,
264 tokenizer,
265 calib_config: None,
266 }
267 }
268
269 pub fn with_calibration(
271 mut self,
272 calib_config: &'a sokuto_core::contract::calibration::CalibrationConfig,
273 ) -> Self {
274 self.calib_config = Some(calib_config);
275 self
276 }
277}
278
279impl<'a> HierarchicalCoarseScorer for ModelDrivenCoarseScorer<'a> {
280 fn score_coarse(
281 &self,
282 state: &str,
283 instructions: &str,
284 coarse_categories: &IndexMap<String, String>,
285 ) -> Result<IndexMap<String, f64>> {
286 let default_calib = sokuto_core::contract::calibration::CalibrationConfig::default();
287 let calib = self.calib_config.unwrap_or(&default_calib);
288
289 let question = Question {
290 question_type: QuestionType::Choice,
291 instructions: instructions.to_string(),
292 criteria: Some(sokuto_core::schema::Criteria::Map(
293 coarse_categories.clone(),
294 )),
295 };
296
297 let answer =
298 self.engine
299 .predict_single_question(self.tokenizer, state, &question, calib)?;
300
301 answer.probabilities.ok_or_else(|| {
302 RuntimeError::InvalidQuestion(
303 "大分類推論結果に確率分布が含まれていません。".to_string(),
304 )
305 })
306 }
307}
308
309#[derive(Debug, Clone)]
314pub struct LexicalCoarseScorer<'a> {
315 tokenizer: &'a JevTokenizer,
317}
318
319impl<'a> LexicalCoarseScorer<'a> {
320 pub fn new(tokenizer: &'a JevTokenizer) -> Self {
322 Self { tokenizer }
323 }
324}
325
326impl CoarseScorer for LexicalCoarseScorer<'_> {
327 fn score_candidates(
328 &self,
329 query: &str,
330 candidates: &IndexMap<String, String>,
331 ) -> Result<Vec<f32>> {
332 let query_encoding = self
334 .tokenizer
335 .inner()
336 .encode(query, false)
337 .map_err(|e| RuntimeError::TokenizerEncodeError(e.to_string()))?;
338 let query_ids = query_encoding.get_ids();
339
340 let mut query_counts = std::collections::HashMap::with_capacity(query_ids.len());
341 for &id in query_ids {
342 *query_counts.entry(id).or_insert(0usize) += 1;
343 }
344
345 let mut scores = Vec::with_capacity(candidates.len());
346
347 for (key, desc) in candidates {
348 let candidate_text = format!("{key} {desc}");
350 let cand_encoding = self
351 .tokenizer
352 .inner()
353 .encode(candidate_text, false)
354 .map_err(|e| RuntimeError::TokenizerEncodeError(e.to_string()))?;
355 let cand_ids = cand_encoding.get_ids();
356
357 if cand_ids.is_empty() {
358 scores.push(0.0);
359 continue;
360 }
361
362 let mut cand_counts = std::collections::HashMap::with_capacity(cand_ids.len());
363 for &id in cand_ids {
364 *cand_counts.entry(id).or_insert(0usize) += 1;
365 }
366
367 let mut dot_product = 0.0f32;
369 for (id, count) in &cand_counts {
370 if let Some(q_count) = query_counts.get(id) {
371 dot_product += (*count as f32) * (*q_count as f32);
372 }
373 }
374
375 let cand_len_sq: f32 = cand_counts.values().map(|c| (c * c) as f32).sum();
376 let query_len_sq: f32 = query_counts.values().map(|c| (c * c) as f32).sum();
377
378 let norm = (cand_len_sq * query_len_sq).sqrt();
379 let score = if norm > 1e-12 {
380 dot_product / norm
381 } else {
382 0.0
383 };
384
385 scores.push(score);
386 }
387
388 Ok(scores)
389 }
390}
391
392#[derive(Debug, Clone)]
394pub struct EmbeddingCoarseScorer<'a> {
395 cache: &'a CandidateEmbeddingCache,
397 query_vector: &'a [f32],
399}
400
401impl<'a> EmbeddingCoarseScorer<'a> {
402 pub fn new(cache: &'a CandidateEmbeddingCache, query_vector: &'a [f32]) -> Self {
404 Self {
405 cache,
406 query_vector,
407 }
408 }
409}
410
411impl CoarseScorer for EmbeddingCoarseScorer<'_> {
412 fn score_candidates(
413 &self,
414 _query: &str,
415 candidates: &IndexMap<String, String>,
416 ) -> Result<Vec<f32>> {
417 let mut scores = Vec::with_capacity(candidates.len());
418
419 for key in candidates.keys() {
420 if let Some(cand_vec) = self.cache.get(key) {
421 let sim = CandidateEmbeddingCache::cosine_similarity(self.query_vector, cand_vec);
422 scores.push(sim);
423 } else {
424 scores.push(0.0);
426 }
427 }
428
429 Ok(scores)
430 }
431}
432
433#[derive(Debug, Clone, PartialEq)]
435pub struct FilteredCandidates {
436 pub selected: IndexMap<String, String>,
438 pub excluded_keys: Vec<String>,
440 pub total_candidates: usize,
442}
443
444pub fn filter_top_candidates(
457 query: &str,
458 candidates: &IndexMap<String, String>,
459 config: &CoarseToFineConfig,
460 scorer: &impl CoarseScorer,
461) -> Result<FilteredCandidates> {
462 let total_candidates = candidates.len();
463
464 if total_candidates <= config.top_m {
466 return Ok(FilteredCandidates {
467 selected: candidates.clone(),
468 excluded_keys: Vec::new(),
469 total_candidates,
470 });
471 }
472
473 let mut pinned_indices = Vec::new();
475 let mut candidate_entries: Vec<(usize, &String, &String)> =
476 Vec::with_capacity(total_candidates);
477
478 for (orig_idx, (key, desc)) in candidates.iter().enumerate() {
479 if config.preserve_negative && is_negative_candidate(key, desc, &config.negative_keys) {
480 pinned_indices.push(orig_idx);
481 } else {
482 candidate_entries.push((orig_idx, key, desc));
483 }
484 }
485
486 let target_top_m = config.top_m.max(1);
488 if pinned_indices.len() > target_top_m {
489 pinned_indices.truncate(target_top_m);
490 }
491 let remaining_quota = target_top_m.saturating_sub(pinned_indices.len());
492
493 let mut selected_indices = pinned_indices;
495
496 if remaining_quota > 0 && !candidate_entries.is_empty() {
497 let mut unpinned_map = IndexMap::with_capacity(candidate_entries.len());
498
499 for &(_, key, desc) in &candidate_entries {
500 unpinned_map.insert(key.clone(), desc.clone());
501 }
502
503 let scores = scorer.score_candidates(query, &unpinned_map)?;
504
505 let mut scored_items: Vec<(f32, usize)> = candidate_entries
507 .iter()
508 .zip(scores.iter())
509 .map(|(&(orig_idx, _, _), &score)| {
510 let s = if score.is_finite() {
512 score
513 } else {
514 f32::NEG_INFINITY
515 };
516 (s, orig_idx)
517 })
518 .collect();
519
520 let select_k = remaining_quota.min(scored_items.len());
522 if select_k < scored_items.len() {
523 scored_items.select_nth_unstable_by(select_k - 1, |a, b| {
525 b.0.partial_cmp(&a.0)
526 .unwrap_or(std::cmp::Ordering::Equal)
527 .then_with(|| a.1.cmp(&b.1))
528 });
529 scored_items.truncate(select_k);
530 }
531
532 for (_, orig_idx) in scored_items {
533 selected_indices.push(orig_idx);
534 }
535 }
536
537 selected_indices.sort_unstable();
539 selected_indices.dedup();
540
541 let mut selected_map = IndexMap::with_capacity(selected_indices.len());
542 let mut excluded_keys = Vec::with_capacity(total_candidates - selected_indices.len());
543
544 let mut selected_idx_set = std::collections::HashSet::with_capacity(selected_indices.len());
545 for &idx in &selected_indices {
546 selected_idx_set.insert(idx);
547 }
548
549 for (orig_idx, (key, desc)) in candidates.iter().enumerate() {
550 if selected_idx_set.contains(&orig_idx) {
551 selected_map.insert(key.clone(), desc.clone());
552 } else {
553 excluded_keys.push(key.clone());
554 }
555 }
556
557 Ok(FilteredCandidates {
558 selected: selected_map,
559 excluded_keys,
560 total_candidates,
561 })
562}
563
564pub fn reconstruct_probabilities(
573 original_keys: &[String],
574 sub_probabilities: &IndexMap<String, f64>,
575) -> IndexMap<String, f64> {
576 let mut full_probabilities = IndexMap::with_capacity(original_keys.len());
577
578 for key in original_keys {
579 let prob = sub_probabilities.get(key).copied().unwrap_or(0.0);
580 full_probabilities.insert(key.clone(), prob);
581 }
582
583 full_probabilities
584}
585
586#[cfg(test)]
587mod tests {
588 use super::*;
589
590 struct MockScorer {
592 scores: Vec<f32>,
593 }
594
595 impl CoarseScorer for MockScorer {
596 fn score_candidates(
597 &self,
598 _query: &str,
599 candidates: &IndexMap<String, String>,
600 ) -> Result<Vec<f32>> {
601 assert_eq!(self.scores.len(), candidates.len());
602 Ok(self.scores.clone())
603 }
604 }
605
606 #[test]
607 fn test_negative_candidate_detection() {
608 let neg_keys = vec!["none".to_string(), "other".to_string()];
609 assert!(is_negative_candidate("None", "該当なし", &neg_keys));
610 assert!(is_negative_candidate("other", "その他", &neg_keys));
611 assert!(is_negative_candidate(
612 "custom",
613 "上記のいずれにも該当しない",
614 &neg_keys
615 ));
616 assert!(!is_negative_candidate("card_lost", "カード紛失", &neg_keys));
617 }
618
619 #[test]
620 fn test_candidate_embedding_cache() {
621 let mut cache = CandidateEmbeddingCache::new();
622 assert!(cache.is_empty());
623
624 cache.insert("opt1", vec![3.0, 4.0]).expect("挿入失敗。");
625 assert_eq!(cache.len(), 1);
626 assert_eq!(cache.dimension(), 2);
627
628 let vec = cache.get("opt1").unwrap();
629 assert!((vec[0] - 0.6).abs() < 1e-6);
631 assert!((vec[1] - 0.8).abs() < 1e-6);
632
633 let sim = CandidateEmbeddingCache::cosine_similarity(vec, &[0.6, 0.8]);
635 assert!((sim - 1.0).abs() < 1e-6);
636 }
637
638 #[test]
639 fn test_filter_top_candidates_pinning_and_determinism() {
640 let mut candidates = IndexMap::new();
641 candidates.insert("card_issue".to_string(), "カード発行".to_string()); candidates.insert("card_lost".to_string(), "カード紛失".to_string()); candidates.insert("loan_apply".to_string(), "ローン申込".to_string()); candidates.insert("loan_repay".to_string(), "ローン返済".to_string()); candidates.insert(
647 "other".to_string(),
648 "上記のいずれにも該当しない".to_string(),
649 ); let config = CoarseToFineConfig {
652 threshold: 3,
653 top_m: 3, preserve_negative: true,
655 negative_keys: vec!["other".to_string()],
656 };
657
658 let scorer = MockScorer {
661 scores: vec![0.1, 0.9, 0.5, 0.5],
662 };
663
664 let result = filter_top_candidates("カード", &candidates, &config, &scorer).unwrap();
665
666 assert_eq!(result.selected.len(), 3);
667 assert_eq!(result.total_candidates, 5);
668
669 assert!(result.selected.contains_key("other"));
671 assert!(result.selected.contains_key("card_lost"));
673 assert!(result.selected.contains_key("loan_apply"));
675 assert!(!result.selected.contains_key("loan_repay"));
676 assert!(!result.selected.contains_key("card_issue"));
677
678 let keys: Vec<&String> = result.selected.keys().collect();
680 assert_eq!(keys, vec!["card_lost", "loan_apply", "other"]);
681 }
682
683 #[test]
684 fn test_reconstruct_probabilities() {
685 let original_keys = vec![
686 "a".to_string(),
687 "b".to_string(),
688 "c".to_string(),
689 "d".to_string(),
690 ];
691 let mut sub_probs = IndexMap::new();
692 sub_probs.insert("b".to_string(), 0.7);
693 sub_probs.insert("d".to_string(), 0.3);
694
695 let full = reconstruct_probabilities(&original_keys, &sub_probs);
696 assert_eq!(full.len(), 4);
697 assert_eq!(full.get("a"), Some(&0.0));
698 assert_eq!(full.get("b"), Some(&0.7));
699 assert_eq!(full.get("c"), Some(&0.0));
700 assert_eq!(full.get("d"), Some(&0.3));
701 }
702
703 #[test]
704 fn test_filter_top_candidates_excessive_negative_keys_guard() {
705 let mut candidates = IndexMap::new();
706 for i in 0..5 {
708 candidates.insert(format!("none_{i}"), format!("該当なし {i}"));
709 }
710 for i in 0..5 {
711 candidates.insert(format!("regular_{i}"), format!("通常候補 {i}"));
712 }
713
714 let config = CoarseToFineConfig {
716 threshold: 5,
717 top_m: 3,
718 preserve_negative: true,
719 negative_keys: (0..5).map(|i| format!("none_{i}")).collect(),
720 };
721
722 let scorer = MockScorer {
723 scores: vec![1.0; 5],
724 };
725
726 let result = filter_top_candidates("クエリ", &candidates, &config, &scorer).unwrap();
727
728 assert_eq!(result.selected.len(), 3);
730 assert_eq!(result.total_candidates, 10);
731 assert!(result.selected.contains_key("none_0"));
733 assert!(result.selected.contains_key("none_1"));
734 assert!(result.selected.contains_key("none_2"));
735 assert!(!result.selected.contains_key("none_3"));
736 assert!(!result.selected.contains_key("regular_0"));
737 }
738}