



'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation

lowercase__     :  Optional[int]    					=			logging.get_logger(__name__)

lowercase__     :  int    					=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowercase__     :  Union[str, Any]    					=			{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

lowercase__     :  int    					=			{
    'gpt2': 10_24,
    'gpt2-medium': 10_24,
    'gpt2-large': 10_24,
    'gpt2-xl': 10_24,
    'distilgpt2': 10_24,
}





@lru_cache()
def 						snake_case__ ():
    UpperCamelCase__      =			(
        list(range(ord("!"	)					,  ord("~"	) + 1	)	) + list(range(ord("¡"	)					,  ord("¬"	) + 1	)	) + list(range(ord("®"	)					,  ord("ÿ"	) + 1	)	)
    )
    UpperCamelCase__      =			bs[:]
    UpperCamelCase__      =			0
    for b in range(2**8	):
        if b not in bs:
            bs.append(lowerCAmelCase_	)
            cs.append(2**8 + n	)
            n += 1
    UpperCamelCase__      =			[chr(lowerCAmelCase_	) for n in cs]
    return dict(zip(lowerCAmelCase_					,  lowerCAmelCase_	)	)





def 						snake_case__ (lowerCAmelCase_	):
    UpperCamelCase__      =			set()
    UpperCamelCase__      =			word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)	)
        UpperCamelCase__      =			char
    return pairs





class       lowerCamelCase_							(	lowerCAmelCase_  ):






   '''simple docstring'''

   A :     List[str]				=	VOCAB_FILES_NAMES
   A :     Any				=	PRETRAINED_VOCAB_FILES_MAP
   A :     Union[str, Any]				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A :     Optional[Any]				=	["input_ids", "attention_mask"]


   def __init__(  self     ,						__UpperCamelCase     ,						__UpperCamelCase     ,						__UpperCamelCase="replace"     ,						__UpperCamelCase="<|endoftext|>"     ,						__UpperCamelCase="<|endoftext|>"     ,						__UpperCamelCase="<|endoftext|>"     ,						__UpperCamelCase=None     ,						__UpperCamelCase=False     ,						__UpperCamelCase=False     ,						**__UpperCamelCase     ,						):
       UpperCamelCase__      =			AddedToken(__UpperCamelCase     ,						lstrip=__UpperCamelCase     ,						rstrip=__UpperCamelCase ) if isinstance(__UpperCamelCase     ,						__UpperCamelCase ) else bos_token
       UpperCamelCase__      =			AddedToken(__UpperCamelCase     ,						lstrip=__UpperCamelCase     ,						rstrip=__UpperCamelCase ) if isinstance(__UpperCamelCase     ,						__UpperCamelCase ) else eos_token
       UpperCamelCase__      =			AddedToken(__UpperCamelCase     ,						lstrip=__UpperCamelCase     ,						rstrip=__UpperCamelCase ) if isinstance(__UpperCamelCase     ,						__UpperCamelCase ) else unk_token
       UpperCamelCase__      =			AddedToken(__UpperCamelCase     ,						lstrip=__UpperCamelCase     ,						rstrip=__UpperCamelCase ) if isinstance(__UpperCamelCase     ,						__UpperCamelCase ) else pad_token
       super().__init__(
           errors=__UpperCamelCase     ,						unk_token=__UpperCamelCase     ,						bos_token=__UpperCamelCase     ,						eos_token=__UpperCamelCase     ,						pad_token=__UpperCamelCase     ,						add_prefix_space=__UpperCamelCase     ,						add_bos_token=__UpperCamelCase     ,						**__UpperCamelCase     ,						)
       UpperCamelCase__      =			add_bos_token

       with open(__UpperCamelCase     ,						encoding="utf-8" ) as vocab_handle:
           UpperCamelCase__      =			json.load(__UpperCamelCase )
       UpperCamelCase__      =			{v: k for k, v in self.encoder.items()}
       UpperCamelCase__      =			errors  # how to handle errors in decoding
       UpperCamelCase__      =			bytes_to_unicode()
       UpperCamelCase__      =			{v: k for k, v in self.byte_encoder.items()}
       with open(__UpperCamelCase     ,						encoding="utf-8" ) as merges_handle:
           UpperCamelCase__      =			merges_handle.read().split("\n" )[1:-1]
       UpperCamelCase__      =			[tuple(merge.split() ) for merge in bpe_merges]
       UpperCamelCase__      =			dict(zip(__UpperCamelCase     ,						range(len(__UpperCamelCase ) ) ) )
       UpperCamelCase__      =			{}
       UpperCamelCase__      =			add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       UpperCamelCase__      =			re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+" )


   @property
   def 	_UpperCamelCase       (  self ):
       return len(self.encoder )


   def 	_UpperCamelCase       (  self ):
       return dict(self.encoder     ,						**self.added_tokens_encoder )


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase ):
       if token in self.cache:
           return self.cache[token]
       UpperCamelCase__      =			tuple(__UpperCamelCase )
       UpperCamelCase__      =			get_pairs(__UpperCamelCase )

       if not pairs:
           return token

       while True:
           UpperCamelCase__      =			min(__UpperCamelCase     ,						key=lambda __UpperCamelCase : self.bpe_ranks.get(__UpperCamelCase     ,						float("inf" ) ) )
           if bigram not in self.bpe_ranks:
               break
           UpperCamelCase__    ,  UpperCamelCase__      =			bigram
           UpperCamelCase__      =			[]
           UpperCamelCase__      =			0
           while i < len(__UpperCamelCase ):
               try:
                   UpperCamelCase__      =			word.index(__UpperCamelCase     ,						__UpperCamelCase )
               except ValueError:
                   new_word.extend(word[i:] )
                   break
               else:
                   new_word.extend(word[i:j] )
                   UpperCamelCase__      =			j

               if word[i] == first and i < len(__UpperCamelCase ) - 1 and word[i + 1] == second:
                   new_word.append(first + second )
                   i += 2
               else:
                   new_word.append(word[i] )
                   i += 1
           UpperCamelCase__      =			tuple(__UpperCamelCase )
           UpperCamelCase__      =			new_word
           if len(__UpperCamelCase ) == 1:
               break
           else:
               UpperCamelCase__      =			get_pairs(__UpperCamelCase )
       UpperCamelCase__      =			" ".join(__UpperCamelCase )
       UpperCamelCase__      =			word
       return word


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase     ,						__UpperCamelCase=None ):
       if self.add_bos_token:
           UpperCamelCase__      =			[self.bos_token_id]
       else:
           UpperCamelCase__      =			[]

       UpperCamelCase__      =			bos_token_ids + token_ids_a

       if token_ids_a is None:
           return output

       return output + bos_token_ids + token_ids_a


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase     ,						__UpperCamelCase = None     ,						__UpperCamelCase = False ):
       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__UpperCamelCase     ,						token_ids_a=__UpperCamelCase     ,						already_has_special_tokens=__UpperCamelCase )

       if not self.add_bos_token:
           return super().get_special_tokens_mask(
               token_ids_a=__UpperCamelCase     ,						token_ids_a=__UpperCamelCase     ,						already_has_special_tokens=__UpperCamelCase )

       if token_ids_a is None:
           return [1] + ([0] * len(__UpperCamelCase ))
       return [1] + ([0] * len(__UpperCamelCase )) + [1] + ([0] * len(__UpperCamelCase ))


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase ):
       UpperCamelCase__      =			[]
       for token in re.findall(self.pat     ,						__UpperCamelCase ):
           UpperCamelCase__      =			"".join(
               self.byte_encoder[b] for b in token.encode("utf-8" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCamelCase ).split(" " ) )
       return bpe_tokens


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase ):
       return self.encoder.get(__UpperCamelCase     ,						self.encoder.get(self.unk_token ) )


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase ):
       return self.decoder.get(__UpperCamelCase )


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase ):
       UpperCamelCase__      =			"".join(__UpperCamelCase )
       UpperCamelCase__      =			bytearray([self.byte_decoder[c] for c in text] ).decode("utf-8"     ,						errors=self.errors )
       return text


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase     ,						__UpperCamelCase = None ):
       if not os.path.isdir(__UpperCamelCase ):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
           return
       UpperCamelCase__      =			os.path.join(
           __UpperCamelCase     ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )
       UpperCamelCase__      =			os.path.join(
           __UpperCamelCase     ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"] )

       with open(__UpperCamelCase     ,						"w"     ,						encoding="utf-8" ) as f:
           f.write(json.dumps(self.encoder     ,						indent=2     ,						sort_keys=__UpperCamelCase     ,						ensure_ascii=__UpperCamelCase ) + "\n" )

       UpperCamelCase__      =			0
       with open(__UpperCamelCase     ,						"w"     ,						encoding="utf-8" ) as writer:
           writer.write("#version: 0.2\n" )
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,						key=lambda __UpperCamelCase : kv[1] ):
               if index != token_index:
                   logger.warning(
                       f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                       " Please check that the tokenizer is not corrupted!" )
                   UpperCamelCase__      =			token_index
               writer.write(" ".join(__UpperCamelCase ) + "\n" )
               index += 1

       return vocab_file, merge_file


   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase     ,						__UpperCamelCase=False     ,						**__UpperCamelCase ):
       UpperCamelCase__      =			kwargs.pop("add_prefix_space"     ,						self.add_prefix_space )
       if is_split_into_words or add_prefix_space:
           UpperCamelCase__      =			" " + text
       return (text, kwargs)







   def 	_UpperCamelCase       (  self     ,						__UpperCamelCase ):
       UpperCamelCase__      =			[]
       for is_user, text in conversation.iter_texts():
           input_ids.extend(self.encode(__UpperCamelCase     ,						add_special_tokens=__UpperCamelCase ) + [self.eos_token_id] )
       if len(__UpperCamelCase ) > self.model_max_length:
           UpperCamelCase__      =			input_ids[-self.model_max_length :]
       return input_ids









'''simple docstring'''







import unittest

from transformers import load_tool
from transformers.tools.agent_types import AGENT_TYPE_MAPPING

from .test_tools_common import ToolTesterMixin, output_types

class       lowerCamelCase_							(	unittest.TestCase					,    lowerCAmelCase_  ):






   '''simple docstring'''

   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			load_tool("translation" )
       self.tool.setup()
       UpperCamelCase__      =			load_tool("translation"     ,						remote=__UpperCamelCase )


   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			self.tool("Hey, what's up?"     ,						src_lang="English"     ,						tgt_lang="French" )
       self.assertEqual(__UpperCamelCase     ,						"- Hé, comment ça va?" )


   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			self.remote_tool("Hey, what's up?"     ,						src_lang="English"     ,						tgt_lang="French" )
       self.assertEqual(__UpperCamelCase     ,						"- Hé, comment ça va?" )


   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			self.tool(text="Hey, what's up?"     ,						src_lang="English"     ,						tgt_lang="French" )
       self.assertEqual(__UpperCamelCase     ,						"- Hé, comment ça va?" )


   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			self.remote_tool(text="Hey, what's up?"     ,						src_lang="English"     ,						tgt_lang="French" )
       self.assertEqual(__UpperCamelCase     ,						"- Hé, comment ça va?" )


   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			["Hey, what's up?", "English", "Spanish"]
       UpperCamelCase__      =			self.tool(*__UpperCamelCase )

       # There is a single output
       if len(self.tool.outputs ) == 1:
           UpperCamelCase__      =			[outputs]

       self.assertListEqual(output_types(__UpperCamelCase )     ,						self.tool.outputs )


   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			["Hey, what's up?", "English", "Spanish"]
       UpperCamelCase__      =			self.tool(*__UpperCamelCase )

       if not isinstance(__UpperCamelCase     ,						__UpperCamelCase ):
           UpperCamelCase__      =			[outputs]

       self.assertEqual(len(__UpperCamelCase )     ,						len(self.tool.outputs ) )

       for output, output_type in zip(__UpperCamelCase     ,						self.tool.outputs ):
           UpperCamelCase__      =			AGENT_TYPE_MAPPING[output_type]
           self.assertTrue(isinstance(__UpperCamelCase     ,						__UpperCamelCase ) )







   def 	_UpperCamelCase       (  self ):
       UpperCamelCase__      =			["Hey, what's up?", "English", "Spanish"]

       UpperCamelCase__      =			[]

       for _input, input_type in zip(__UpperCamelCase     ,						self.tool.inputs ):
           if isinstance(__UpperCamelCase     ,						__UpperCamelCase ):
               _inputs.append([AGENT_TYPE_MAPPING[_input_type](_input ) for _input_type in input_type] )
           else:
               _inputs.append(AGENT_TYPE_MAPPING[input_type](_input ) )

        # Should not raise an error
       UpperCamelCase__      =			self.tool(*__UpperCamelCase )

       if not isinstance(__UpperCamelCase     ,						__UpperCamelCase ):
           UpperCamelCase__      =			[outputs]

       self.assertEqual(len(__UpperCamelCase )     ,						len(self.tool.outputs ) )




