"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
    if is_torch_available():
        import torch
    if is_tf_available():
        import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__UpperCamelCase								=    logging.get_logger(__name__)

__UpperCamelCase								=    {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__UpperCamelCase								=    {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

__UpperCamelCase								=    {
    '''Salesforce/codegen-350M-mono''': 2048,
}







@lru_cache()
def       UpperCAmelCase					(	)       ->   Optional[Any]:
     snake_case_       =      (
         list(range(ord('!'      )		,      ord('~'      ) + 1      )      ) + list(range(ord('¡'      )		,      ord('¬'      ) + 1      )      ) + list(range(ord('®'      )		,      ord('ÿ'      ) + 1      )      )
     )
     snake_case_       =      bs[:]
     snake_case_       =      0
     for b in range(2**8      ):
          if b not in bs:
               bs.append(UpperCAmelCase      )
               cs.append(2**8 + n      )
               n += 1
     snake_case_       =      [chr(UpperCAmelCase      ) for n in cs]
     return dict(zip(UpperCAmelCase		,      UpperCAmelCase      )      )







def       UpperCAmelCase					(	UpperCAmelCase      )       ->   str:
     snake_case_       =      set()
     snake_case_       =      word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)      )
          snake_case_       =      char
     return pairs




class 				UpperCamelCase      (       lowerCAmelCase__     ):
       SCREAMING_SNAKE_CASE_				=   VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE_				=   PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE_				=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE_				=   ["input_ids", "attention_mask"]







       def __init__(				self,							lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__="replace",							lowerCAmelCase__="<|endoftext|>",							lowerCAmelCase__="<|endoftext|>",							lowerCAmelCase__="<|endoftext|>",							lowerCAmelCase__=None,							lowerCAmelCase__=False,							lowerCAmelCase__=False,							**lowerCAmelCase__,							)   ->   int:
            snake_case_       =      AddedToken(lowerCAmelCase__,							lstrip=lowerCAmelCase__,							rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,							lowerCAmelCase__) else bos_token
            snake_case_       =      AddedToken(lowerCAmelCase__,							lstrip=lowerCAmelCase__,							rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,							lowerCAmelCase__) else eos_token
            snake_case_       =      AddedToken(lowerCAmelCase__,							lstrip=lowerCAmelCase__,							rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,							lowerCAmelCase__) else unk_token
            snake_case_       =      AddedToken(lowerCAmelCase__,							lstrip=lowerCAmelCase__,							rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__,							lowerCAmelCase__) else pad_token
            super().__init__(
                errors=lowerCAmelCase__,							unk_token=lowerCAmelCase__,							bos_token=lowerCAmelCase__,							eos_token=lowerCAmelCase__,							pad_token=lowerCAmelCase__,							add_prefix_space=lowerCAmelCase__,							add_bos_token=lowerCAmelCase__,							**lowerCAmelCase__,							)
            snake_case_       =      add_bos_token

            with open(lowerCAmelCase__,							encoding='utf-8') as vocab_handle:
                 snake_case_       =      json.load(lowerCAmelCase__)
            snake_case_       =      {v: k for k, v in self.encoder.items()}
            snake_case_       =      errors  # how to handle errors in decoding
            snake_case_       =      bytes_to_unicode()
            snake_case_       =      {v: k for k, v in self.byte_encoder.items()}
            with open(lowerCAmelCase__,							encoding='utf-8') as merges_handle:
                 snake_case_       =      merges_handle.read().split('\n')[1:-1]
            snake_case_       =      [tuple(merge.split()) for merge in bpe_merges]
            snake_case_       =      dict(zip(lowerCAmelCase__,							range(len(lowerCAmelCase__))))
            snake_case_       =      {}
            snake_case_       =      add_prefix_space

            # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
            snake_case_       =      re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')







       @property
       def 	a_		(				self)   ->   Any:
            return len(self.encoder)







       def 	a_		(				self)   ->   Optional[int]:
            return dict(self.encoder,							**self.added_tokens_encoder)







       def 	a_		(				self,							lowerCAmelCase__)   ->   Tuple:
            if token in self.cache:
                 return self.cache[token]
            snake_case_       =      tuple(lowerCAmelCase__)
            snake_case_       =      get_pairs(lowerCAmelCase__)

            if not pairs:
                 return token

            while True:
                 snake_case_       =      min(lowerCAmelCase__,							key=lambda lowerCAmelCase__: self.bpe_ranks.get(lowerCAmelCase__,							float('inf')))
                 if bigram not in self.bpe_ranks:
                      break
                 snake_case_ ,			snake_case_       =      bigram
                 snake_case_       =      []
                 snake_case_       =      0
                 while i < len(lowerCAmelCase__):
                      try:
                           snake_case_       =      word.index(lowerCAmelCase__,							lowerCAmelCase__)
                      except ValueError:
                           new_word.extend(word[i:])
                           break
                      else:
                           new_word.extend(word[i:j])
                           snake_case_       =      j

                      if word[i] == first and i < len(lowerCAmelCase__) - 1 and word[i + 1] == second:
                           new_word.append(first + second)
                           i += 2
                      else:
                           new_word.append(word[i])
                           i += 1
                 snake_case_       =      tuple(lowerCAmelCase__)
                 snake_case_       =      new_word
                 if len(lowerCAmelCase__) == 1:
                      break
                 else:
                      snake_case_       =      get_pairs(lowerCAmelCase__)
            snake_case_       =      ' '.join(lowerCAmelCase__)
            snake_case_       =      word
            return word







       def 	a_		(				self,							lowerCAmelCase__,							lowerCAmelCase__=None)   ->   Any:
            if self.add_bos_token:
                 snake_case_       =      [self.bos_token_id]
            else:
                 snake_case_       =      []

            snake_case_       =      bos_token_ids + token_ids_a

            if token_ids_a is None:
                 return output

            return output + bos_token_ids + token_ids_a







       def 	a_		(				self,							lowerCAmelCase__)   ->   Union[str, Any]:
            snake_case_       =      []
            for token in re.findall(self.pat,							lowerCAmelCase__):
                 snake_case_       =      ''.join(
                     self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__).split(' '))
            return bpe_tokens







       def 	a_		(				self,							lowerCAmelCase__)   ->   List[Any]:
            return self.encoder.get(lowerCAmelCase__,							self.encoder.get(self.unk_token))







       def 	a_		(				self,							lowerCAmelCase__)   ->   Tuple:
            return self.decoder.get(lowerCAmelCase__)







       def 	a_		(				self,							lowerCAmelCase__)   ->   Optional[Any]:
            snake_case_       =      ''.join(lowerCAmelCase__)
            snake_case_       =      bytearray([self.byte_decoder[c] for c in text]).decode('utf-8',							errors=self.errors)
            return text







       def 	a_		(				self,							lowerCAmelCase__,							lowerCAmelCase__ = None)   ->   Tuple[str]:
            if not os.path.isdir(lowerCAmelCase__):
                 logger.error(f'Vocabulary path ({save_directory}) should be a directory')
                 return
            snake_case_       =      os.path.join(
                lowerCAmelCase__,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
            snake_case_       =      os.path.join(
                lowerCAmelCase__,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

            with open(lowerCAmelCase__,							'w',							encoding='utf-8') as f:
                 f.write(json.dumps(self.encoder,							indent=2,							sort_keys=lowerCAmelCase__,							ensure_ascii=lowerCAmelCase__) + '\n')

            snake_case_       =      0
            with open(lowerCAmelCase__,							'w',							encoding='utf-8') as writer:
                 writer.write('#version: 0.2\n')
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),							key=lambda lowerCAmelCase__: kv[1]):
                      if index != token_index:
                           logger.warning(
                               f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                               ' Please check that the tokenizer is not corrupted!')
                           snake_case_       =      token_index
                      writer.write(' '.join(lowerCAmelCase__) + '\n')
                      index += 1

            return vocab_file, merge_file







       def 	a_		(				self,							lowerCAmelCase__,							lowerCAmelCase__=False,							**lowerCAmelCase__)   ->   Union[str, Any]:
            snake_case_       =      kwargs.pop('add_prefix_space',							self.add_prefix_space)
            if is_split_into_words or add_prefix_space:
                 snake_case_       =      ' ' + text
            return (text, kwargs)







       def 	a_		(				self,							lowerCAmelCase__,							lowerCAmelCase__ = False,							lowerCAmelCase__ = None,							lowerCAmelCase__ = None,							**lowerCAmelCase__,							)   ->   str:
            snake_case_       =      super()._decode(
                token_ids=lowerCAmelCase__,							skip_special_tokens=lowerCAmelCase__,							clean_up_tokenization_spaces=lowerCAmelCase__,							**lowerCAmelCase__,							)

            if truncate_before_pattern is not None and len(lowerCAmelCase__) > 0:
                 snake_case_       =      self.truncate(lowerCAmelCase__,							lowerCAmelCase__)

            return decoded_text







       def 	a_		(				self,							lowerCAmelCase__,							lowerCAmelCase__)   ->   Optional[int]:
            def find_re(lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__):
                 snake_case_       =      pattern.search(lowerCAmelCase__,							lowerCAmelCase__)
                 return m.start() if m else -1

            snake_case_       =      [re.compile(lowerCAmelCase__,							re.MULTILINE) for pattern in truncate_before_pattern]

            snake_case_       =      list(re.finditer('^print',							lowerCAmelCase__,							re.MULTILINE))

            if len(lowerCAmelCase__) > 1:
                 snake_case_       =      completion[: prints[1].start()]

            snake_case_       =      list(re.finditer('^def',							lowerCAmelCase__,							re.MULTILINE))

            if len(lowerCAmelCase__) > 1:
                 snake_case_       =      completion[: defs[1].start()]

            snake_case_       =      0

            snake_case_       =      [
                pos for pos in [find_re(lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__) for terminal in terminals] if pos != -1
            ]

            if len(lowerCAmelCase__) > 0:
                 return completion[: min(lowerCAmelCase__)]
            else:
                 return completion


"""simple docstring"""


import functools
from contextlib import contextmanager

import bert_score
from packaging import version

import datasets







@contextmanager
def       UpperCAmelCase					(	)       ->   str:
     def filter_log(UpperCAmelCase      ):
          return False if "This IS expected if you are initializing" in record.msg else True

     snake_case_       =      datasets.utils.logging.get_logger('transformers.modeling_utils'      )
     logger.addFilter(UpperCAmelCase      )
     try:
          yield
     finally:
          logger.removeFilter(UpperCAmelCase      )


__UpperCamelCase								=    '''\
@inproceedings{bert-score,
  title={BERTScore: Evaluating Text Generation with BERT},
  author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi},
  booktitle={International Conference on Learning Representations},
  year={2020},
  url={https://openreview.net/forum?id=SkeHuCVFDr}
}
'''

__UpperCamelCase								=    '''\
BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference
sentences by cosine similarity.
It has been shown to correlate with human judgment on sentence-level and system-level evaluation.
Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language
generation tasks.

See the project\'s README at https://github.com/Tiiiger/bert_score#readme for more information.
'''

__UpperCamelCase								=    '''
BERTScore Metrics with the hashcode from a source against one or more references.

Args:
    predictions (list of str): Prediction/candidate sentences.
    references (list of str or list of list of str): Reference sentences.
    lang (str): Language of the sentences; required (e.g. \'en\').
    model_type (str): Bert specification, default using the suggested
        model for the target language; has to specify at least one of
        `model_type` or `lang`.
    num_layers (int): The layer of representation to use,
        default using the number of layers tuned on WMT16 correlation data.
    verbose (bool): Turn on intermediate status update.
    idf (bool or dict): Use idf weighting; can also be a precomputed idf_dict.
    device (str): On which the contextual embedding model will be allocated on.
        If this argument is None, the model lives on cuda:0 if cuda is available.
    nthreads (int): Number of threads.
    batch_size (int): Bert score processing batch size,
        at least one of `model_type` or `lang`. `lang` needs to be
        specified when `rescale_with_baseline` is True.
    rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline.
    baseline_path (str): Customized baseline file.
    use_fast_tokenizer (bool): `use_fast` parameter passed to HF tokenizer. New in version 0.3.10.

Returns:
    precision: Precision.
    recall: Recall.
    f1: F1 score.
    hashcode: Hashcode of the library.

Examples:

    >>> predictions = ["hello there", "general kenobi"]
    >>> references = ["hello there", "general kenobi"]
    >>> bertscore = datasets.load_metric("bertscore")
    >>> results = bertscore.compute(predictions=predictions, references=references, lang="en")
    >>> print([round(v, 2) for v in results["f1"]])
    [1.0, 1.0]
'''




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,	_KWARGS_DESCRIPTION     )
class 				UpperCamelCase      (       datasets.Metric     ):







       def 	a_		(				self)   ->   Dict:
            return datasets.MetricInfo(
                description=_DESCRIPTION,							citation=_CITATION,							homepage='https://github.com/Tiiiger/bert_score',							inputs_description=_KWARGS_DESCRIPTION,							features=datasets.Features(
                    {
                        'predictions': datasets.Value('string',							id='sequence'),
                        'references': datasets.Sequence(datasets.Value('string',							id='sequence'),							id='references'),
                    }),							codebase_urls=['https://github.com/Tiiiger/bert_score'],							reference_urls=[
                    'https://github.com/Tiiiger/bert_score',
                    'https://arxiv.org/abs/1904.09675',
                ],							)







       def 	a_		(				self,							lowerCAmelCase__,							lowerCAmelCase__,							lowerCAmelCase__=None,							lowerCAmelCase__=None,							lowerCAmelCase__=None,							lowerCAmelCase__=False,							lowerCAmelCase__=False,							lowerCAmelCase__=None,							lowerCAmelCase__=64,							lowerCAmelCase__=4,							lowerCAmelCase__=False,							lowerCAmelCase__=False,							lowerCAmelCase__=None,							lowerCAmelCase__=False,							)   ->   Union[str, Any]:
            snake_case_       =      bert_score.utils.get_hash
            snake_case_       =      bert_score.BERTScorer

            if version.parse(bert_score.__version__) >= version.parse('0.3.10'):
                 snake_case_       =      functools.partial(lowerCAmelCase__,							use_fast_tokenizer=lowerCAmelCase__)
                 snake_case_       =      functools.partial(lowerCAmelCase__,							use_fast_tokenizer=lowerCAmelCase__)
            elif use_fast_tokenizer:
                 raise ImportWarning(
                     'To use a fast tokenizer, the module `bert-score>=0.3.10` is required, and the current version of `bert-score` doesn\'t match this condition.\n'
                     'You can install it with `pip install "bert-score>=0.3.10"`.')

            if model_type is None:
                 assert lang is not None, "either lang or model_type should be specified"
                 snake_case_       =      bert_score.utils.langamodel[lang.lower()]

            if num_layers is None:
                 snake_case_       =      bert_score.utils.modelalayers[model_type]

            snake_case_       =      get_hash(
                model=lowerCAmelCase__,							num_layers=lowerCAmelCase__,							idf=lowerCAmelCase__,							rescale_with_baseline=lowerCAmelCase__,							use_custom_baseline=baseline_path is not None,							)

            with filter_logging_context():
                 if not hasattr(self,							'cached_bertscorer') or self.cached_bertscorer.hash != hashcode:
                      snake_case_       =      scorer(
                          model_type=lowerCAmelCase__,							num_layers=lowerCAmelCase__,							batch_size=lowerCAmelCase__,							nthreads=lowerCAmelCase__,							all_layers=lowerCAmelCase__,							idf=lowerCAmelCase__,							device=lowerCAmelCase__,							lang=lowerCAmelCase__,							rescale_with_baseline=lowerCAmelCase__,							baseline_path=lowerCAmelCase__,							)

            ((snake_case_) ,			(snake_case_) ,			(snake_case_))       =      self.cached_bertscorer.score(
                cands=lowerCAmelCase__,							refs=lowerCAmelCase__,							verbose=lowerCAmelCase__,							batch_size=lowerCAmelCase__,							)
            snake_case_       =      {
                'precision': P.tolist(),
                'recall': R.tolist(),
                'f1': F.tolist(),
                'hashcode': hashcode,
            }
            return output_dict







       def 	a_		(				self,							lowerCAmelCase__=None,							lowerCAmelCase__=None,							**lowerCAmelCase__)   ->   List[str]:
            # References can be strings or lists of strings
            # Let's change strings to lists of strings with one element
            if references is not None:
                 snake_case_       =      [[ref] if isinstance(lowerCAmelCase__,							lowerCAmelCase__) else ref for ref in references]
            super().add_batch(predictions=lowerCAmelCase__,							references=lowerCAmelCase__,							**lowerCAmelCase__)







       def 	a_		(				self,							lowerCAmelCase__=None,							lowerCAmelCase__=None,							**lowerCAmelCase__)   ->   List[str]:
            # References can be strings or lists of strings
            # Let's change strings to lists of strings with one element
            if isinstance(lowerCAmelCase__,							lowerCAmelCase__):
                 snake_case_       =      [reference]
            super().add(prediction=lowerCAmelCase__,							reference=lowerCAmelCase__,							**lowerCAmelCase__)

