
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_A							=       logging.get_logger(__name__)

_A							=       {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_A							=       {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

_A							=       {
    '''Salesforce/codegen-350M-mono''': 2_048,
}




@lru_cache()
def 					lowerCamelCase__					(    )		->						Union[str, Any]:
       UpperCamelCase_											=							(
           list(range(ord("""!"""  )						,   ord("""~"""  ) + 1  )  ) + list(range(ord("""¡"""  )						,   ord("""¬"""  ) + 1  )  ) + list(range(ord("""®"""  )						,   ord("""ÿ"""  ) + 1  )  )
       )
       UpperCamelCase_											=							bs[:]
       UpperCamelCase_											=							0
       for b in range(2**8  ):
              if b not in bs:
                     bs.append(a_  )
                     cs.append(2**8 + n  )
                     n += 1
       UpperCamelCase_											=							[chr(a_  ) for n in cs]
       return dict(zip(a_						,   a_  )  )




def 					lowerCamelCase__					(    a__	:	Union[str, Any]  )		->						Optional[int]:
       UpperCamelCase_											=							set()
       UpperCamelCase_											=							word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)  )
              UpperCamelCase_											=							char
       return pairs







class 		lowercase_							(							__SCREAMING_SNAKE_CASE					):
 A__    :					Tuple    =     VOCAB_FILES_NAMES
 A__    :					List[str]    =     PRETRAINED_VOCAB_FILES_MAP
 A__    :					int    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 A__    :					Tuple    =     ["input_ids", "attention_mask"]

 def __init__(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase="replace"  ,						__UpperCamelCase="<|endoftext|>"  ,						__UpperCamelCase="<|endoftext|>"  ,						__UpperCamelCase="<|endoftext|>"  ,						__UpperCamelCase=None  ,						__UpperCamelCase=False  ,						__UpperCamelCase=False  ,						**__UpperCamelCase  ,						):




        """simple docstring"""


        UpperCamelCase_											=							AddedToken(__UpperCamelCase  ,						lstrip=__UpperCamelCase  ,						rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) else bos_token
        UpperCamelCase_											=							AddedToken(__UpperCamelCase  ,						lstrip=__UpperCamelCase  ,						rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) else eos_token
        UpperCamelCase_											=							AddedToken(__UpperCamelCase  ,						lstrip=__UpperCamelCase  ,						rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) else unk_token
        UpperCamelCase_											=							AddedToken(__UpperCamelCase  ,						lstrip=__UpperCamelCase  ,						rstrip=__UpperCamelCase  ) if isinstance(__UpperCamelCase  ,						__UpperCamelCase  ) else pad_token
        super().__init__(
            errors=__UpperCamelCase  ,						unk_token=__UpperCamelCase  ,						bos_token=__UpperCamelCase  ,						eos_token=__UpperCamelCase  ,						pad_token=__UpperCamelCase  ,						add_prefix_space=__UpperCamelCase  ,						add_bos_token=__UpperCamelCase  ,						**__UpperCamelCase  ,						)
        UpperCamelCase_											=							add_bos_token

        with open(__UpperCamelCase  ,						encoding="""utf-8"""  ) as vocab_handle:
               UpperCamelCase_											=							json.load(__UpperCamelCase  )
        UpperCamelCase_											=							{v: k for k, v in self.encoder.items()}
        UpperCamelCase_											=							errors  # how to handle errors in decoding
        UpperCamelCase_											=							bytes_to_unicode()
        UpperCamelCase_											=							{v: k for k, v in self.byte_encoder.items()}
        with open(__UpperCamelCase  ,						encoding="""utf-8"""  ) as merges_handle:
               UpperCamelCase_											=							merges_handle.read().split("""\n"""  )[1:-1]
        UpperCamelCase_											=							[tuple(merge.split()  ) for merge in bpe_merges]
        UpperCamelCase_											=							dict(zip(__UpperCamelCase  ,						range(len(__UpperCamelCase  )  )  )  )
        UpperCamelCase_											=							{}
        UpperCamelCase_											=							add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        UpperCamelCase_											=							re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""  )

 @property
 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        return len(self.encoder  )

 def lowerCamelCase_					(   self  ):




        """simple docstring"""


        return dict(self.encoder  ,						**self.added_tokens_encoder  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        if token in self.cache:
               return self.cache[token]
        UpperCamelCase_											=							tuple(__UpperCamelCase  )
        UpperCamelCase_											=							get_pairs(__UpperCamelCase  )

        if not pairs:
               return token

        while True:
               UpperCamelCase_											=							min(__UpperCamelCase  ,						key=lambda __UpperCamelCase  : self.bpe_ranks.get(__UpperCamelCase  ,						float("""inf"""  )  )  )
               if bigram not in self.bpe_ranks:
                      break
               UpperCamelCase_							,					UpperCamelCase_											=							bigram
               UpperCamelCase_											=							[]
               UpperCamelCase_											=							0
               while i < len(__UpperCamelCase  ):
                      try:
                             UpperCamelCase_											=							word.index(__UpperCamelCase  ,						__UpperCamelCase  )
                      except ValueError:
                             new_word.extend(word[i:]  )
                             break
                      else:
                             new_word.extend(word[i:j]  )
                             UpperCamelCase_											=							j

                      if word[i] == first and i < len(__UpperCamelCase  ) - 1 and word[i + 1] == second:
                             new_word.append(first + second  )
                             i += 2
                      else:
                             new_word.append(word[i]  )
                             i += 1
               UpperCamelCase_											=							tuple(__UpperCamelCase  )
               UpperCamelCase_											=							new_word
               if len(__UpperCamelCase  ) == 1:
                      break
               else:
                      UpperCamelCase_											=							get_pairs(__UpperCamelCase  )
        UpperCamelCase_											=							""" """.join(__UpperCamelCase  )
        UpperCamelCase_											=							word
        return word

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase=None  ):




        """simple docstring"""


        if self.add_bos_token:
               UpperCamelCase_											=							[self.bos_token_id]
        else:
               UpperCamelCase_											=							[]

        UpperCamelCase_											=							bos_token_ids + token_ids_a

        if token_ids_a is None:
               return output

        return output + bos_token_ids + token_ids_a

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							[]
        for token in re.findall(self.pat  ,						__UpperCamelCase  ):
               UpperCamelCase_											=							"""""".join(
                   self.byte_encoder[b] for b in token.encode("""utf-8"""  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCamelCase  ).split(""" """  )  )
        return bpe_tokens

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        return self.encoder.get(__UpperCamelCase  ,						self.encoder.get(self.unk_token  )  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        return self.decoder.get(__UpperCamelCase  )

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							"""""".join(__UpperCamelCase  )
        UpperCamelCase_											=							bytearray([self.byte_decoder[c] for c in text]  ).decode("""utf-8"""  ,						errors=self.errors  )
        return text

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase = None  ):




        """simple docstring"""


        if not os.path.isdir(__UpperCamelCase  ):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
               return
        UpperCamelCase_											=							os.path.join(
            __UpperCamelCase  ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
        UpperCamelCase_											=							os.path.join(
            __UpperCamelCase  ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

        with open(__UpperCamelCase  ,						"""w"""  ,						encoding="""utf-8"""  ) as f:
               f.write(json.dumps(self.encoder  ,						indent=2  ,						sort_keys=__UpperCamelCase  ,						ensure_ascii=__UpperCamelCase  ) + """\n"""  )

        UpperCamelCase_											=							0
        with open(__UpperCamelCase  ,						"""w"""  ,						encoding="""utf-8"""  ) as writer:
               writer.write("""#version: 0.2\n"""  )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,						key=lambda __UpperCamelCase  : kv[1]  ):
                      if index != token_index:
                             logger.warning(
                                 f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                 """ Please check that the tokenizer is not corrupted!"""  )
                             UpperCamelCase_											=							token_index
                      writer.write(""" """.join(__UpperCamelCase  ) + """\n"""  )
                      index += 1

        return vocab_file, merge_file

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase=False  ,						**__UpperCamelCase  ):




        """simple docstring"""


        UpperCamelCase_											=							kwargs.pop("""add_prefix_space"""  ,						self.add_prefix_space  )
        if is_split_into_words or add_prefix_space:
               UpperCamelCase_											=							""" """ + text
        return (text, kwargs)

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase = False  ,						__UpperCamelCase = None  ,						__UpperCamelCase = None  ,						**__UpperCamelCase  ,						):




        """simple docstring"""


        UpperCamelCase_											=							super()._decode(
            token_ids=__UpperCamelCase  ,						skip_special_tokens=__UpperCamelCase  ,						clean_up_tokenization_spaces=__UpperCamelCase  ,						**__UpperCamelCase  ,						)

        if truncate_before_pattern is not None and len(__UpperCamelCase  ) > 0:
               UpperCamelCase_											=							self.truncate(__UpperCamelCase  ,						__UpperCamelCase  )

        return decoded_text

 def lowerCamelCase_					(   self  ,						__UpperCamelCase  ,						__UpperCamelCase  ):




        """simple docstring"""


        def find_re(__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase  ):
               UpperCamelCase_											=							pattern.search(__UpperCamelCase  ,						__UpperCamelCase  )
               return m.start() if m else -1

        UpperCamelCase_											=							[re.compile(__UpperCamelCase  ,						re.MULTILINE  ) for pattern in truncate_before_pattern]

        UpperCamelCase_											=							list(re.finditer("""^print"""  ,						__UpperCamelCase  ,						re.MULTILINE  )  )

        if len(__UpperCamelCase  ) > 1:
               UpperCamelCase_											=							completion[: prints[1].start()]

        UpperCamelCase_											=							list(re.finditer("""^def"""  ,						__UpperCamelCase  ,						re.MULTILINE  )  )

        if len(__UpperCamelCase  ) > 1:
               UpperCamelCase_											=							completion[: defs[1].start()]

        UpperCamelCase_											=							0

        UpperCamelCase_											=							[
            pos for pos in [find_re(__UpperCamelCase  ,						__UpperCamelCase  ,						__UpperCamelCase  ) for terminal in terminals] if pos != -1
        ]

        if len(__UpperCamelCase  ) > 0:
               return completion[: min(__UpperCamelCase  )]
        else:
               return completion






from __future__ import annotations

from collections.abc import Iterator
from typing import Any

class 					__UpperCamelCase					:







	def __init__( self:			Optional[Any]	,			__UpperCamelCase:			Any   ):

						'''simple docstring'''





						__magic_name__				=     data
						__magic_name__				=     None







	def __str__( self:			Optional[int]   ):

						'''simple docstring'''





						return F'{self.data}'




class 					__UpperCamelCase					:







	def __init__( self:			List[Any]   ):

						'''simple docstring'''





						__magic_name__				=     None
						__magic_name__				=     None







	def __iter__( self:			str   ):

						'''simple docstring'''





						__magic_name__				=     self.front
						while node:
											yield node.data
											__magic_name__				=     node.next







	def __len__( self:			Any   ):

						'''simple docstring'''





						return len(tuple(iter(self   )   )   )







	def __str__( self:			List[str]   ):

						'''simple docstring'''





						return " <- ".join(str(__UpperCamelCase   ) for item in self   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Any   ):

						'''simple docstring'''





						return len(self   ) == 0







	def 						_SCREAMING_SNAKE_CASE				( self:			Any	,			__UpperCamelCase:			Any   ):

						'''simple docstring'''





						__magic_name__				=     Node(__UpperCamelCase   )
						if self.is_empty():
											__magic_name__				=     __magic_name__				=     node
						else:
											assert isinstance(self.rear	,			__UpperCamelCase   )
											__magic_name__				=     node
											__magic_name__				=     node







	def 						_SCREAMING_SNAKE_CASE				( self:			Any   ):

						'''simple docstring'''





						if self.is_empty():
											raise IndexError('dequeue from empty queue'   )
						assert isinstance(self.front	,			__UpperCamelCase   )
						__magic_name__				=     self.front
						__magic_name__				=     self.front.next
						if self.front is None:
											__magic_name__				=     None
						return node.data







	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple   ):

						'''simple docstring'''





						__magic_name__				=     __magic_name__				=     None


if __name__ == "__main__":
						from doctest import testmod

						testmod()




