
'''simple docstring'''

import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_     			=       logging.get_logger(__name__)

UpperCamelCase_     			=       {
    """src_vocab_file""": """vocab-src.json""",
    """tgt_vocab_file""": """vocab-tgt.json""",
    """merges_file""": """merges.txt""",
}

UpperCamelCase_     			=       {
    """src_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"""
    },
    """tgt_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"""
    },
    """merges_file""": {"""stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"""},
}

UpperCamelCase_     			=       {"""stas/tiny-wmt19-en-de""": 10_24}
UpperCamelCase_     			=       {
    """stas/tiny-wmt19-en-de""": {
        """langs""": ["""en""", """de"""],
        """model_max_length""": 10_24,
        """special_tokens_map_file""": None,
        """full_tokenizer_file""": None,
    }
}


def 	_UpperCAmelCase				(  _lowerCamelCase						:       str		)    ->						Optional[int]:
  _lowerCAmelCase       :  Optional[Any]   				=					set()
  _lowerCAmelCase       :  int   				=					word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)		)
    _lowerCAmelCase       :  Dict   				=					char
  return pairs


def 	_UpperCAmelCase				(  _lowerCamelCase						:       Any		)    ->						Optional[Any]:
  _lowerCAmelCase       :  Optional[int]   				=					text.replace("""，"""  ,      ""","""		)
  _lowerCAmelCase       :  int   				=					re.sub(R"""。\s*"""  ,      """. """  ,      _lowerCamelCase		)
  _lowerCAmelCase       :  int   				=					text.replace("""、"""  ,      ""","""		)
  _lowerCAmelCase       :  List[Any]   				=					text.replace("""”"""  ,      """\""""		)
  _lowerCAmelCase       :  Tuple   				=					text.replace("""“"""  ,      """\""""		)
  _lowerCAmelCase       :  Dict   				=					text.replace("""∶"""  ,      """:"""		)
  _lowerCAmelCase       :  str   				=					text.replace("""："""  ,      """:"""		)
  _lowerCAmelCase       :  Dict   				=					text.replace("""？"""  ,      """?"""		)
  _lowerCAmelCase       :  List[str]   				=					text.replace("""《"""  ,      """\""""		)
  _lowerCAmelCase       :  List[str]   				=					text.replace("""》"""  ,      """\""""		)
  _lowerCAmelCase       :  List[str]   				=					text.replace("""）"""  ,      """)"""		)
  _lowerCAmelCase       :  str   				=					text.replace("""！"""  ,      """!"""		)
  _lowerCAmelCase       :  Optional[Any]   				=					text.replace("""（"""  ,      """("""		)
  _lowerCAmelCase       :  Tuple   				=					text.replace("""；"""  ,      """;"""		)
  _lowerCAmelCase       :  Tuple   				=					text.replace("""１"""  ,      """1"""		)
  _lowerCAmelCase       :  Any   				=					text.replace("""」"""  ,      """\""""		)
  _lowerCAmelCase       :  str   				=					text.replace("""「"""  ,      """\""""		)
  _lowerCAmelCase       :  List[Any]   				=					text.replace("""０"""  ,      """0"""		)
  _lowerCAmelCase       :  Tuple   				=					text.replace("""３"""  ,      """3"""		)
  _lowerCAmelCase       :  Optional[Any]   				=					text.replace("""２"""  ,      """2"""		)
  _lowerCAmelCase       :  List[Any]   				=					text.replace("""５"""  ,      """5"""		)
  _lowerCAmelCase       :  int   				=					text.replace("""６"""  ,      """6"""		)
  _lowerCAmelCase       :  List[str]   				=					text.replace("""９"""  ,      """9"""		)
  _lowerCAmelCase       :  Optional[int]   				=					text.replace("""７"""  ,      """7"""		)
  _lowerCAmelCase       :  Dict   				=					text.replace("""８"""  ,      """8"""		)
  _lowerCAmelCase       :  Optional[int]   				=					text.replace("""４"""  ,      """4"""		)
  _lowerCAmelCase       :  List[Any]   				=					re.sub(R"""．\s*"""  ,      """. """  ,      _lowerCamelCase		)
  _lowerCAmelCase       :  List[Any]   				=					text.replace("""～"""  ,      """~"""		)
  _lowerCAmelCase       :  Union[str, Any]   				=					text.replace("""’"""  ,      """'"""		)
  _lowerCAmelCase       :  Any   				=					text.replace("""…"""  ,      """..."""		)
  _lowerCAmelCase       :  Tuple   				=					text.replace("""━"""  ,      """-"""		)
  _lowerCAmelCase       :  Dict   				=					text.replace("""〈"""  ,      """<"""		)
  _lowerCAmelCase       :  Any   				=					text.replace("""〉"""  ,      """>"""		)
  _lowerCAmelCase       :  List[str]   				=					text.replace("""【"""  ,      """["""		)
  _lowerCAmelCase       :  List[str]   				=					text.replace("""】"""  ,      """]"""		)
  _lowerCAmelCase       :  List[Any]   				=					text.replace("""％"""  ,      """%"""		)
  return text


def 	_UpperCAmelCase				(  _lowerCamelCase						:       Union[str, Any]		)    ->						List[str]:
  _lowerCAmelCase       :  int   				=					[]
  for char in text:
    _lowerCAmelCase       :  Union[str, Any]   				=					unicodedata.category(_lowerCamelCase		)
    if cat.startswith("""C"""		):
      continue
    output.append(_lowerCamelCase		)
  return "".join(_lowerCamelCase		)



class   a_  (_a    ):
       __lowerCAmelCase    :				Tuple    =      VOCAB_FILES_NAMES
       __lowerCAmelCase    :				Optional[Any]    =      PRETRAINED_VOCAB_FILES_MAP
       __lowerCAmelCase    :				List[str]    =      PRETRAINED_INIT_CONFIGURATION
       __lowerCAmelCase    :				int    =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowerCAmelCase    :				Any    =      ["""input_ids""", """attention_mask"""]




       def __init__(     self   ,   snake_case_=None   ,   snake_case_=None   ,   snake_case_=None   ,   snake_case_=None   ,   snake_case_=False   ,   snake_case_="<unk>"   ,   snake_case_="<s>"   ,   snake_case_="</s>"   ,   snake_case_="<pad>"   ,   **snake_case_   ,   ):
         super().__init__(
             langs=snake_case_   ,   src_vocab_file=snake_case_   ,   tgt_vocab_file=snake_case_   ,   merges_file=snake_case_   ,   do_lower_case=snake_case_   ,   unk_token=snake_case_   ,   bos_token=snake_case_   ,   sep_token=snake_case_   ,   pad_token=snake_case_   ,   **snake_case_   ,   )

         try:
           import sacremoses
         except ImportError:
           raise ImportError(
               """You need to install sacremoses to use XLMTokenizer. """
               """See https://pypi.org/project/sacremoses/ for installation."""  )

         _lowerCAmelCase       :  Any   				=					sacremoses

         _lowerCAmelCase       :  Optional[int]   				=					src_vocab_file
         _lowerCAmelCase       :  Dict   				=					tgt_vocab_file
         _lowerCAmelCase       :  Union[str, Any]   				=					merges_file
         _lowerCAmelCase       :  str   				=					do_lower_case

         # cache of sm.MosesPunctNormalizer instance
         _lowerCAmelCase       :  int   				=					{}
         # cache of sm.MosesTokenizer instance
         _lowerCAmelCase       :  Union[str, Any]   				=					{}
         _lowerCAmelCase       :  Union[str, Any]   				=					{}

         if langs and len(snake_case_  ) == 2:
           _lowerCAmelCase				,					_lowerCAmelCase       :  List[str]   				=					langs
         else:
           raise ValueError(
               f'arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '
               """Usually that means that tokenizer can't find a mapping for the given model path """
               """in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."""  )

         with open(snake_case_   ,   encoding="""utf-8"""  ) as src_vocab_handle:
           _lowerCAmelCase       :  List[Any]   				=					json.load(snake_case_  )
         with open(snake_case_   ,   encoding="""utf-8"""  ) as tgt_vocab_handle:
           _lowerCAmelCase       :  Union[str, Any]   				=					json.load(snake_case_  )
           _lowerCAmelCase       :  Any   				=					{v: k for k, v in tgt_vocab.items()}
         with open(snake_case_   ,   encoding="""utf-8"""  ) as merges_handle:
           _lowerCAmelCase       :  str   				=					merges_handle.read().split("""\n"""  )[:-1]
         _lowerCAmelCase       :  Union[str, Any]   				=					[tuple(merge.split()[:2]  ) for merge in merges]
         _lowerCAmelCase       :  str   				=					dict(zip(snake_case_   ,   range(len(snake_case_  )  )  )  )
         _lowerCAmelCase       :  Tuple   				=					{}




       def 							__UpperCamelCase       (     self  ):
         return self.get_src_vocab()




       @property
       def 							__UpperCamelCase       (     self  ):
         return self.src_vocab_size




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_  ):
         if lang not in self.cache_moses_punct_normalizer:
           _lowerCAmelCase       :  Optional[Any]   				=					self.sm.MosesPunctNormalizer(lang=snake_case_  )
           _lowerCAmelCase       :  Tuple   				=					punct_normalizer
         return self.cache_moses_punct_normalizer[lang].normalize(snake_case_  )




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_  ):
         if lang not in self.cache_moses_tokenizer:
           _lowerCAmelCase       :  List[str]   				=					self.sm.MosesTokenizer(lang=snake_case_  )
           _lowerCAmelCase       :  Tuple   				=					moses_tokenizer
         return self.cache_moses_tokenizer[lang].tokenize(
             snake_case_   ,   aggressive_dash_splits=snake_case_   ,   return_str=snake_case_   ,   escape=snake_case_  )




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_  ):
         if lang not in self.cache_moses_detokenizer:
           _lowerCAmelCase       :  Optional[Any]   				=					self.sm.MosesDetokenizer(lang=snake_case_  )
           _lowerCAmelCase       :  Union[str, Any]   				=					moses_detokenizer
         return self.cache_moses_detokenizer[lang].detokenize(snake_case_  )




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_  ):
         _lowerCAmelCase       :  int   				=					replace_unicode_punct(snake_case_  )
         _lowerCAmelCase       :  int   				=					self.moses_punct_norm(snake_case_   ,   snake_case_  )
         _lowerCAmelCase       :  Union[str, Any]   				=					remove_non_printing_char(snake_case_  )
         return text




       @property
       def 							__UpperCamelCase       (     self  ):
         return len(self.encoder  )




       @property
       def 							__UpperCamelCase       (     self  ):
         return len(self.decoder  )




       def 							__UpperCamelCase       (     self  ):
         return dict(self.encoder   ,   **self.added_tokens_encoder  )




       def 							__UpperCamelCase       (     self  ):
         return dict(self.decoder   ,   **self.added_tokens_decoder  )




       def 							__UpperCamelCase       (     self   ,   snake_case_  ):
         _lowerCAmelCase       :  Tuple   				=					tuple(token[:-1]  ) + (token[-1] + """</w>""",)
         if token in self.cache:
           return self.cache[token]
         _lowerCAmelCase       :  Tuple   				=					get_pairs(snake_case_  )

         if not pairs:
           return token + "</w>"

         while True:
           _lowerCAmelCase       :  str   				=					min(snake_case_   ,   key=lambda snake_case_  : self.bpe_ranks.get(snake_case_   ,   float("""inf"""  )  )  )
           if bigram not in self.bpe_ranks:
             break
           _lowerCAmelCase				,					_lowerCAmelCase       :  List[str]   				=					bigram
           _lowerCAmelCase       :  Optional[int]   				=					[]
           _lowerCAmelCase       :  List[Any]   				=					0
           while i < len(snake_case_  ):
             try:
               _lowerCAmelCase       :  Optional[int]   				=					word.index(snake_case_   ,   snake_case_  )
             except ValueError:
               new_word.extend(word[i:]  )
               break
             else:
               new_word.extend(word[i:j]  )
               _lowerCAmelCase       :  Dict   				=					j

             if word[i] == first and i < len(snake_case_  ) - 1 and word[i + 1] == second:
               new_word.append(first + second  )
               i += 2
             else:
               new_word.append(word[i]  )
               i += 1
           _lowerCAmelCase       :  List[str]   				=					tuple(snake_case_  )
           _lowerCAmelCase       :  Optional[int]   				=					new_word
           if len(snake_case_  ) == 1:
             break
           else:
             _lowerCAmelCase       :  List[Any]   				=					get_pairs(snake_case_  )
         _lowerCAmelCase       :  Union[str, Any]   				=					""" """.join(snake_case_  )
         if word == "\n  </w>":
           _lowerCAmelCase       :  Dict   				=					"""\n</w>"""
         _lowerCAmelCase       :  List[Any]   				=					word
         return word




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_="en"   ,   snake_case_=False  ):
         _lowerCAmelCase       :  Union[str, Any]   				=					self.src_lang

         if self.do_lower_case:
           _lowerCAmelCase       :  str   				=					text.lower()

         if bypass_tokenizer:
           _lowerCAmelCase       :  Any   				=					text.split()
         else:
           _lowerCAmelCase       :  Dict   				=					self.moses_pipeline(snake_case_   ,   lang=snake_case_  )
           _lowerCAmelCase       :  str   				=					self.moses_tokenize(snake_case_   ,   lang=snake_case_  )

         _lowerCAmelCase       :  int   				=					[]
         for token in text:
           if token:
             split_tokens.extend(list(self.bpe(snake_case_  ).split(""" """  )  )  )

         return split_tokens




       def 							__UpperCamelCase       (     self   ,   snake_case_  ):
         return self.encoder.get(snake_case_   ,   self.encoder.get(self.unk_token  )  )




       def 							__UpperCamelCase       (     self   ,   snake_case_  ):
         return self.decoder.get(snake_case_   ,   self.unk_token  )




       def 							__UpperCamelCase       (     self   ,   snake_case_  ):
         _lowerCAmelCase       :  Tuple   				=					[t.replace(""" """   ,   """"""  ).replace("""</w>"""   ,   """ """  ) for t in tokens]
         _lowerCAmelCase       :  Optional[Any]   				=					"""""".join(snake_case_  ).split()
         # detokenize
         _lowerCAmelCase       :  Union[str, Any]   				=					self.moses_detokenize(snake_case_   ,   self.tgt_lang  )
         return text




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_ = None  ):
         _lowerCAmelCase       :  str   				=					[self.sep_token_id]

         # no bos used in fairseq
         if token_ids_a is None:
           return token_ids_a + sep
         return token_ids_a + sep + token_ids_a + sep




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_ = None   ,   snake_case_ = False  ):

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=snake_case_   ,   token_ids_a=snake_case_   ,   already_has_special_tokens=snake_case_  )
         # no bos used in fairseq
         if token_ids_a is not None:
           return ([0] * len(snake_case_  )) + [1] + ([0] * len(snake_case_  )) + [1]
         return ([0] * len(snake_case_  )) + [1]




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_ = None  ):
         _lowerCAmelCase       :  List[str]   				=					[self.sep_token_id]

         # no bos used in fairseq
         if token_ids_a is None:
           return len(token_ids_a + sep  ) * [0]
         return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




       def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_ = None  ):
         if not os.path.isdir(snake_case_  ):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory'  )
           return

         _lowerCAmelCase       :  List[Any]   				=					os.path.join(
             snake_case_   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""src_vocab_file"""]  )
         _lowerCAmelCase       :  List[str]   				=					os.path.join(
             snake_case_   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""tgt_vocab_file"""]  )
         _lowerCAmelCase       :  Union[str, Any]   				=					os.path.join(
             snake_case_   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

         with open(snake_case_   ,   """w"""   ,   encoding="""utf-8"""  ) as f:
           f.write(json.dumps(self.encoder   ,   indent=2   ,   sort_keys=snake_case_   ,   ensure_ascii=snake_case_  ) + """\n"""  )

         with open(snake_case_   ,   """w"""   ,   encoding="""utf-8"""  ) as f:
           _lowerCAmelCase       :  Optional[Any]   				=					{v: k for k, v in self.decoder.items()}
           f.write(json.dumps(snake_case_   ,   indent=2   ,   sort_keys=snake_case_   ,   ensure_ascii=snake_case_  ) + """\n"""  )

         _lowerCAmelCase       :  List[Any]   				=					0
         with open(snake_case_   ,   """w"""   ,   encoding="""utf-8"""  ) as writer:
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,   key=lambda snake_case_  : kv[1]  ):
             if index != token_index:
               logger.warning(
                   f'Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'
                   """ Please check that the tokenizer is not corrupted!"""  )
               _lowerCAmelCase       :  Any   				=					token_index
             writer.write(""" """.join(snake_case_  ) + """\n"""  )
             index += 1

         return src_vocab_file, tgt_vocab_file, merges_file




       def __getstate__(     self  ):
         _lowerCAmelCase       :  List[Any]   				=					self.__dict__.copy()
         _lowerCAmelCase       :  List[Any]   				=					None
         return state







       def __setstate__(     self   ,   snake_case_  ):
         _lowerCAmelCase       :  List[Any]   				=					d

         try:
           import sacremoses
         except ImportError:
           raise ImportError(
               """You need to install sacremoses to use XLMTokenizer. """
               """See https://pypi.org/project/sacremoses/ for installation."""  )

         _lowerCAmelCase       :  List[Any]   				=					sacremoses




'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCamelCase_     			=       {
    """configuration_deformable_detr""": ["""DEFORMABLE_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DeformableDetrConfig"""],
}

try:
     if not is_vision_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     UpperCamelCase_     			=       ["""DeformableDetrFeatureExtractor"""]
     UpperCamelCase_     			=       ["""DeformableDetrImageProcessor"""]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     UpperCamelCase_     			=       [
         """DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """DeformableDetrForObjectDetection""",
         """DeformableDetrModel""",
         """DeformableDetrPreTrainedModel""",
     ]


if TYPE_CHECKING:
     from .configuration_deformable_detr import DEFORMABLE_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP, DeformableDetrConfig

     try:
          if not is_vision_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .feature_extraction_deformable_detr import DeformableDetrFeatureExtractor
          from .image_processing_deformable_detr import DeformableDetrImageProcessor

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_deformable_detr import (
              DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
              DeformableDetrForObjectDetection,
              DeformableDetrModel,
              DeformableDetrPreTrainedModel,
          )

else:
     import sys

     UpperCamelCase_     			=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


