


"""simple docstring"""




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__SCREAMING_SNAKE_CASE					=						logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE					=						{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__SCREAMING_SNAKE_CASE					=						{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

__SCREAMING_SNAKE_CASE					=						{
    """Salesforce/codegen-350M-mono""": 20_48,
}



@lru_cache()
def 						__a      (  ):





       """simple docstring"""







       _a          =				(
           list(range(ord("!"					),		ord("~"					) + 1					)					) + list(range(ord("¡"					),		ord("¬"					) + 1					)					) + list(range(ord("®"					),		ord("ÿ"					) + 1					)					)
       )
       _a          =				bs[:]
       _a          =				0
       for b in range(2**8					):
              if b not in bs:
                     bs.append(a					)
                     cs.append(2**8 + n					)
                     n += 1
       _a          =				[chr(a					) for n in cs]
       return dict(zip(a,		a					)					)



def 						__a      (  a					):





       """simple docstring"""







       _a          =				set()
       _a          =				word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)					)
              _a          =				char
       return pairs





class __snake_case		(					_SCREAMING_SNAKE_CASE      ):







   """simple docstring"""




   lowerCAmelCase_				:   str     	=			VOCAB_FILES_NAMES
   lowerCAmelCase_				:   Union[str, Any]     	=			PRETRAINED_VOCAB_FILES_MAP
   lowerCAmelCase_				:   List[Any]     	=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCAmelCase_				:   Tuple     	=			['input_ids', 'attention_mask']




   def __init__(		self							:Optional[Any]				,  UpperCamelCase__							:Optional[int]				,  UpperCamelCase__							:Any				,  UpperCamelCase__							:List[str]="replace"				,  UpperCamelCase__							:Dict="<|endoftext|>"				,  UpperCamelCase__							:Tuple="<|endoftext|>"				,  UpperCamelCase__							:Optional[Any]="<|endoftext|>"				,  UpperCamelCase__							:Tuple=None				,  UpperCamelCase__							:Tuple=False				,  UpperCamelCase__							:Tuple=False				,  **UpperCamelCase__							:List[Any]				,  ):
          _a          =				AddedToken(UpperCamelCase__				,  lstrip=UpperCamelCase__				,  rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__				,  UpperCamelCase__       ) else bos_token
          _a          =				AddedToken(UpperCamelCase__				,  lstrip=UpperCamelCase__				,  rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__				,  UpperCamelCase__       ) else eos_token
          _a          =				AddedToken(UpperCamelCase__				,  lstrip=UpperCamelCase__				,  rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__				,  UpperCamelCase__       ) else unk_token
          _a          =				AddedToken(UpperCamelCase__				,  lstrip=UpperCamelCase__				,  rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__				,  UpperCamelCase__       ) else pad_token
          super().__init__(
              errors=UpperCamelCase__				,  unk_token=UpperCamelCase__				,  bos_token=UpperCamelCase__				,  eos_token=UpperCamelCase__				,  pad_token=UpperCamelCase__				,  add_prefix_space=UpperCamelCase__				,  add_bos_token=UpperCamelCase__				,  **UpperCamelCase__				,  )
          _a          =				add_bos_token

          with open(UpperCamelCase__				,  encoding="utf-8"       ) as vocab_handle:
                 _a          =				json.load(UpperCamelCase__       )
          _a          =				{v: k for k, v in self.encoder.items()}
          _a          =				errors  # how to handle errors in decoding
          _a          =				bytes_to_unicode()
          _a          =				{v: k for k, v in self.byte_encoder.items()}
          with open(UpperCamelCase__				,  encoding="utf-8"       ) as merges_handle:
                 _a          =				merges_handle.read().split("\n"       )[1:-1]
          _a          =				[tuple(merge.split()       ) for merge in bpe_merges]
          _a          =				dict(zip(UpperCamelCase__				,  range(len(UpperCamelCase__       )       )       )       )
          _a          =				{}
          _a          =				add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          _a          =				re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"       )




   @property
   def SCREAMING_SNAKE_CASE_      (		self							:Union[str, Any]       ):
          return len(self.encoder       )




   def SCREAMING_SNAKE_CASE_      (		self							:Tuple       ):
          return dict(self.encoder				,  **self.added_tokens_encoder       )




   def SCREAMING_SNAKE_CASE_      (		self							:str				,  UpperCamelCase__							:str       ):
          if token in self.cache:
                 return self.cache[token]
          _a          =				tuple(UpperCamelCase__       )
          _a          =				get_pairs(UpperCamelCase__       )

          if not pairs:
                 return token

          while True:
                 _a          =				min(UpperCamelCase__				,  key=lambda UpperCamelCase__       : self.bpe_ranks.get(UpperCamelCase__				,  float("inf"       )       )       )
                 if bigram not in self.bpe_ranks:
                        break
                 _a   , _a          =				bigram
                 _a          =				[]
                 _a          =				0
                 while i < len(UpperCamelCase__       ):
                        try:
                               _a          =				word.index(UpperCamelCase__				,  UpperCamelCase__       )
                        except ValueError:
                               new_word.extend(word[i:]       )
                               break
                        else:
                               new_word.extend(word[i:j]       )
                               _a          =				j

                        if word[i] == first and i < len(UpperCamelCase__       ) - 1 and word[i + 1] == second:
                               new_word.append(first + second       )
                               i += 2
                        else:
                               new_word.append(word[i]       )
                               i += 1
                 _a          =				tuple(UpperCamelCase__       )
                 _a          =				new_word
                 if len(UpperCamelCase__       ) == 1:
                        break
                 else:
                        _a          =				get_pairs(UpperCamelCase__       )
          _a          =				" ".join(UpperCamelCase__       )
          _a          =				word
          return word




   def SCREAMING_SNAKE_CASE_      (		self							:int				,  UpperCamelCase__							:int				,  UpperCamelCase__							:Any=None       ):
          if self.add_bos_token:
                 _a          =				[self.bos_token_id]
          else:
                 _a          =				[]

          _a          =				bos_token_ids + token_ids_a

          if token_ids_a is None:
                 return output

          return output + bos_token_ids + token_ids_a




   def SCREAMING_SNAKE_CASE_      (		self							:Optional[int]				,  UpperCamelCase__							:List[str]       ):
          _a          =				[]
          for token in re.findall(self.pat				,  UpperCamelCase__       ):
                 _a          =				"".join(
                     self.byte_encoder[b] for b in token.encode("utf-8"       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__       ).split(" "       )       )
          return bpe_tokens




   def SCREAMING_SNAKE_CASE_      (		self							:Dict				,  UpperCamelCase__							:List[Any]       ):
          return self.encoder.get(UpperCamelCase__				,  self.encoder.get(self.unk_token       )       )




   def SCREAMING_SNAKE_CASE_      (		self							:List[Any]				,  UpperCamelCase__							:Dict       ):
          return self.decoder.get(UpperCamelCase__       )




   def SCREAMING_SNAKE_CASE_      (		self							:Union[str, Any]				,  UpperCamelCase__							:Optional[int]       ):
          _a          =				"".join(UpperCamelCase__       )
          _a          =				bytearray([self.byte_decoder[c] for c in text]       ).decode("utf-8"				,  errors=self.errors       )
          return text




   def SCREAMING_SNAKE_CASE_      (		self							:Tuple				,  UpperCamelCase__							:str				,  UpperCamelCase__							:Optional[str] = None       ):
          if not os.path.isdir(UpperCamelCase__       ):
                 logger.error(f'Vocabulary path ({save_directory}) should be a directory'       )
                 return
          _a          =				os.path.join(
              UpperCamelCase__				,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
          _a          =				os.path.join(
              UpperCamelCase__				,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]       )

          with open(UpperCamelCase__				,  "w"				,  encoding="utf-8"       ) as f:
                 f.write(json.dumps(self.encoder				,  indent=2				,  sort_keys=UpperCamelCase__				,  ensure_ascii=UpperCamelCase__       ) + "\n"       )

          _a          =				0
          with open(UpperCamelCase__				,  "w"				,  encoding="utf-8"       ) as writer:
                 writer.write("#version: 0.2\n"       )
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,  key=lambda UpperCamelCase__       : kv[1]       ):
                        if index != token_index:
                               logger.warning(
                                   f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                   " Please check that the tokenizer is not corrupted!"       )
                               _a          =				token_index
                        writer.write(" ".join(UpperCamelCase__       ) + "\n"       )
                        index += 1

          return vocab_file, merge_file




   def SCREAMING_SNAKE_CASE_      (		self							:Dict				,  UpperCamelCase__							:Dict				,  UpperCamelCase__							:Optional[int]=False				,  **UpperCamelCase__							:List[Any]       ):
          _a          =				kwargs.pop("add_prefix_space"				,  self.add_prefix_space       )
          if is_split_into_words or add_prefix_space:
                 _a          =				" " + text
          return (text, kwargs)




   def SCREAMING_SNAKE_CASE_      (		self							:List[str]				,  UpperCamelCase__							:Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]				,  UpperCamelCase__							:bool = False				,  UpperCamelCase__							:bool = None				,  UpperCamelCase__							:Optional[List[str]] = None				,  **UpperCamelCase__							:int				,  ):
          _a          =				super()._decode(
              token_ids=UpperCamelCase__				,  skip_special_tokens=UpperCamelCase__				,  clean_up_tokenization_spaces=UpperCamelCase__				,  **UpperCamelCase__				,  )

          if truncate_before_pattern is not None and len(UpperCamelCase__       ) > 0:
                 _a          =				self.truncate(UpperCamelCase__				,  UpperCamelCase__       )

          return decoded_text




   def SCREAMING_SNAKE_CASE_      (		self							:Dict				,  UpperCamelCase__							:Dict				,  UpperCamelCase__							:List[Any]       ):
          def find_re(UpperCamelCase__							:Optional[Any]				,  UpperCamelCase__							:str				,  UpperCamelCase__							:List[Any]       ):
                 _a          =				pattern.search(UpperCamelCase__				,  UpperCamelCase__       )
                 return m.start() if m else -1

          _a          =				[re.compile(UpperCamelCase__				,  re.MULTILINE       ) for pattern in truncate_before_pattern]

          _a          =				list(re.finditer("^print"				,  UpperCamelCase__				,  re.MULTILINE       )       )

          if len(UpperCamelCase__       ) > 1:
                 _a          =				completion[: prints[1].start()]

          _a          =				list(re.finditer("^def"				,  UpperCamelCase__				,  re.MULTILINE       )       )

          if len(UpperCamelCase__       ) > 1:
                 _a          =				completion[: defs[1].start()]

          _a          =				0

          _a          =				[
              pos for pos in [find_re(UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__       ) for terminal in terminals] if pos != -1
          ]

          if len(UpperCamelCase__       ) > 0:
                 return completion[: min(UpperCamelCase__       )]
          else:
                 return completion




"""simple docstring"""




from __future__ import annotations



def 						__a      (  a,		a					):





       """simple docstring"""







       assert len(a					) == len(a					)

       _a          =				len(a					)
       _a          =				list(range(a					)					)
       _a          =				[-1] * n  # who the donor has donated to
       _a          =				[-1] * n  # who the recipient has received from
       _a          =				[0] * n

       while unmatched_donors:
              _a          =				unmatched_donors[0]
              _a          =				donor_pref[donor]
              _a          =				donor_preference[num_donations[donor]]
              num_donations[donor] += 1
              _a          =				recipient_pref[recipient]
              _a          =				rec_record[recipient]

              if prev_donor != -1:
                     if rec_preference.index(a					) > rec_preference.index(a					):
                            _a          =				donor
                            _a          =				recipient
                            unmatched_donors.append(a					)
                            unmatched_donors.remove(a					)
              else:
                     _a          =				donor
                     _a          =				recipient
                     unmatched_donors.remove(a					)
       return donor_record
