






"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowercase           =							logging.get_logger(__name__)

lowercase           =							{'''vocab_file''': '''vocab.txt'''}

lowercase           =							{
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

lowercase           =							{
    '''tau/splinter-base''': 512,
    '''tau/splinter-base-qass''': 512,
    '''tau/splinter-large''': 512,
    '''tau/splinter-large-qass''': 512,
}

lowercase           =							{
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}

class     A_				(   _snake_case	):
				UpperCAmelCase__										=					VOCAB_FILES_NAMES
				UpperCAmelCase__										=					PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase__										=					PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase__										=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase__										=					SplinterTokenizer







				def __init__(				self				:    List[Any]    ,     __lowerCamelCase				:    List[str]=None    ,     __lowerCamelCase				:    List[Any]=None    ,     __lowerCamelCase				:    Tuple=True    ,     __lowerCamelCase				:    int="[UNK]"    ,     __lowerCamelCase				:    Dict="[SEP]"    ,     __lowerCamelCase				:    Optional[Any]="[PAD]"    ,     __lowerCamelCase				:    Tuple="[CLS]"    ,     __lowerCamelCase				:    Tuple="[MASK]"    ,     __lowerCamelCase				:    Optional[int]="[QUESTION]"    ,     __lowerCamelCase				:    Any=True    ,     __lowerCamelCase				:    Tuple=None    ,     **__lowerCamelCase				:    List[str]    ,     )					->       int:
										super().__init__(
										    lowerCAmelCase__    ,     tokenizer_file=lowerCAmelCase__    ,     do_lower_case=lowerCAmelCase__    ,     unk_token=lowerCAmelCase__    ,     sep_token=lowerCAmelCase__    ,     pad_token=lowerCAmelCase__    ,     cls_token=lowerCAmelCase__    ,     mask_token=lowerCAmelCase__    ,     tokenize_chinese_chars=lowerCAmelCase__    ,     strip_accents=lowerCAmelCase__    ,     additional_special_tokens=(question_token,)    ,     **lowerCAmelCase__    ,     )

										__magic_name__      =  json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
										if (
										    pre_tok_state.get("lowercase"    ,     lowerCAmelCase__						) != do_lower_case
										    or pre_tok_state.get("strip_accents"    ,     lowerCAmelCase__						) != strip_accents
										):
																__magic_name__      =  getattr(lowerCAmelCase__    ,     pre_tok_state.pop("type"						)						)
																__magic_name__      =  do_lower_case
																__magic_name__      =  strip_accents
																__magic_name__      =  pre_tok_class(**lowerCAmelCase__						)

										__magic_name__      =  do_lower_case







				@property
				def      _snake_case       (				self				:    Optional[int]						)					->       Dict:
										return self.convert_tokens_to_ids(self.question_token						)







				def      _snake_case       (				self				:    Optional[int]    ,     __lowerCamelCase				:    List[int]    ,     __lowerCamelCase				:    Optional[List[int]] = None						)					->       List[int]:
										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										__magic_name__      =  [self.cls_token_id]
										__magic_name__      =  [self.sep_token_id]
										__magic_name__      =  [self.question_token_id] + [self.convert_tokens_to_ids("."						)]
										if self.padding_side == "right":
																# Input is question-then-context
																return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
										else:
																# Input is context-then-question
																return cls + token_ids_a + sep + token_ids_a + question_suffix + sep







				def      _snake_case       (				self				:    Optional[int]    ,     __lowerCamelCase				:    List[int]    ,     __lowerCamelCase				:    Optional[List[int]] = None						)					->       List[int]:
										__magic_name__      =  [self.sep_token_id]
										__magic_name__      =  [self.cls_token_id]
										__magic_name__      =  [self.question_token_id] + [self.convert_tokens_to_ids("."						)]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep						) * [0]

										if self.padding_side == "right":
																# Input is question-then-context
																return len(cls + token_ids_a + question_suffix + sep						) * [0] + len(token_ids_a + sep						) * [1]
										else:
																# Input is context-then-question
																return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + question_suffix + sep						) * [1]







				def      _snake_case       (				self				:    Optional[int]    ,     __lowerCamelCase				:    str    ,     __lowerCamelCase				:    Optional[str] = None						)					->       Tuple[str]:
										__magic_name__      =  self._tokenizer.model.save(lowerCAmelCase__    ,     name=lowerCAmelCase__						)
										return tuple(lowerCAmelCase__						)






from typing import Dict, Optional, Tuple

import torch
def   _lowerCAmelCase(						__A      ):
  UpperCAmelCase          =	boundaries[1] - boundaries[0]
  UpperCAmelCase          =	boundaries + step / 2
  UpperCAmelCase          =	torch.cat([bin_centers, (bin_centers[-1] + step).unsqueeze(-1      )] ,							dim=0      )
  return bin_centers
def   _lowerCAmelCase(						__A ,							__A ,							):
  UpperCAmelCase          =	_calculate_bin_centers(__A      )
  return (
      torch.sum(aligned_distance_error_probs * bin_centers ,							dim=-1      ),
      bin_centers[-1],
  )
def   _lowerCAmelCase(						__A ,							__A = 31 ,							__A = 64 ,							**__A ,							):
  UpperCAmelCase          =	torch.linspace(0 ,							__A ,							steps=(no_bins - 1) ,							device=logits.device      )

  UpperCAmelCase          =	torch.nn.functional.softmax(__A ,							dim=-1      )
  UpperCAmelCase   ,						UpperCAmelCase          =	_calculate_expected_aligned_error(
      alignment_confidence_breaks=__A ,							aligned_distance_error_probs=__A ,							)

  return {
      "aligned_confidence_probs": aligned_confidence_probs,
      "predicted_aligned_error": predicted_aligned_error,
      "max_predicted_aligned_error": max_predicted_aligned_error,
  }





def   _lowerCAmelCase(						__A ,							__A = None ,							__A = 31 ,							__A = 64 ,							__A = 1E-8 ,							**__A ,							):
  if residue_weights is None:
    UpperCAmelCase          =	logits.new_ones(logits.shape[-2]      )

  UpperCAmelCase          =	torch.linspace(0 ,							__A ,							steps=(no_bins - 1) ,							device=logits.device      )

  UpperCAmelCase          =	_calculate_bin_centers(__A      )
  torch.sum(__A      )
  UpperCAmelCase          =	logits.shape[-2]
  UpperCAmelCase          =	max(__A ,							19      )

  UpperCAmelCase          =	1.24 * (clipped_n - 15) ** (1.0 / 3) - 1.8

  UpperCAmelCase          =	torch.nn.functional.softmax(__A ,							dim=-1      )

  UpperCAmelCase          =	1.0 / (1 + (bin_centers**2) / (da**2))
  UpperCAmelCase          =	torch.sum(probs * tm_per_bin ,							dim=-1      )

  UpperCAmelCase          =	residue_weights / (eps + residue_weights.sum())
  UpperCAmelCase          =	torch.sum(predicted_tm_term * normed_residue_mask ,							dim=-1      )

  UpperCAmelCase          =	per_alignment * residue_weights

  UpperCAmelCase          =	(weighted == torch.max(__A      )).nonzero()[0]
  return per_alignment[tuple(__A      )]



