




import argparse
import collections

import jax
import jax.numpy as jnp
import torch
import torch.nn as nn
from clip.model import CLIP
from flax.training import checkpoints
from huggingface_hub import Repository

from transformers import (
    CLIPTokenizer,
    OwlViTConfig,
    OwlViTForObjectDetection,
    OwlViTImageProcessor,
    OwlViTModel,
    OwlViTProcessor,
)


lowercase_			:       int						       =				{
    'vit_b32': {
        'embed_dim': 5_12,
        'image_resolution': 7_68,
        'context_length': 16,
        'vocab_size': 4_94_08,
        'vision_layers': 12,
        'vision_width': 7_68,
        'vision_patch_size': 32,
        'transformer_width': 5_12,
        'transformer_heads': 8,
        'transformer_layers': 12,
    },
    'vit_b16': {
        'embed_dim': 5_12,
        'image_resolution': 7_68,
        'context_length': 16,
        'vocab_size': 4_94_08,
        'vision_layers': 12,
        'vision_width': 7_68,
        'vision_patch_size': 16,
        'transformer_width': 5_12,
        'transformer_heads': 8,
        'transformer_layers': 12,
    },
    'vit_l14': {
        'embed_dim': 7_68,
        'image_resolution': 8_40,
        'context_length': 16,
        'vocab_size': 4_94_08,
        'vision_layers': 24,
        'vision_width': 10_24,
        'vision_patch_size': 14,
        'transformer_width': 7_68,
        'transformer_heads': 12,
        'transformer_layers': 12,
    },
}


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_=""			,					snake_case_="/"		):

						'''simple docstring'''


						_UpperCAmelCase									=  []

						for k, v in params.items():
												_UpperCAmelCase									=  parent_key + sep + k if parent_key else k

												if isinstance(lowercase__			,					collections.MutableMapping		):
																		items.extend(flatten_nested_dict(lowercase__			,					lowercase__			,					sep=lowercase__		).items()		)
												else:
																		items.append((new_key, v)		)
						return dict(lowercase__		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_		):

						'''simple docstring'''


						return jax.tree_util.tree_map(lambda snake_case_		: x.astype(jnp.floataa		) if x.dtype == jnp.bfloataa else x			,					lowercase__		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase									=  pt_attn_layer.in_proj_weight.chunk(3			,					dim=0		)
						_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase									=  pt_attn_layer.in_proj_bias.chunk(3			,					dim=0		)

						_UpperCAmelCase									=  pt_attn_layer.out_proj.weight
						_UpperCAmelCase									=  pt_attn_layer.out_proj.bias

						_UpperCAmelCase									=  q_proj
						_UpperCAmelCase									=  q_proj_bias

						_UpperCAmelCase									=  k_proj
						_UpperCAmelCase									=  k_proj_bias

						_UpperCAmelCase									=  v_proj
						_UpperCAmelCase									=  v_proj_bias

						_UpperCAmelCase									=  out_proj_weights
						_UpperCAmelCase									=  out_proj_bias


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						copy_linear(hf_mlp.fca			,					pt_mlp.c_fc		)
						copy_linear(hf_mlp.fca			,					pt_mlp.c_proj		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  pt_linear.weight
						_UpperCAmelCase									=  pt_linear.bias


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						copy_linear(hf_layer.layer_norma			,					pt_layer.ln_a		)
						copy_linear(hf_layer.layer_norma			,					pt_layer.ln_a		)

						# copy MLP
						copy_mlp(hf_layer.mlp			,					pt_layer.mlp		)

						# copy attn
						copy_attn_layer(hf_layer.self_attn			,					pt_layer.attn		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						for hf_layer, pt_layer in zip(lowercase__			,					lowercase__		):
												copy_layer(lowercase__			,					lowercase__		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  pt_model.token_embedding.weight
						_UpperCAmelCase									=  pt_model.positional_embedding

						# copy layer norm
						copy_linear(hf_encoder.final_layer_norm			,					pt_model.ln_final		)

						# copy hidden layers
						copy_layers(hf_encoder.encoder.layers			,					pt_model.transformer.resblocks		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  pt_model.text_projection.data.T

						# copy text encoder
						copy_encoder(hf_model.text_model			,					lowercase__		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  pt_model.visual.proj.data.T

						# copy layer norms
						copy_linear(hf_model.vision_model.pre_layernorm			,					pt_model.visual.ln_pre		)
						copy_linear(hf_model.vision_model.post_layernorm			,					pt_model.visual.ln_post		)

						# copy embeds
						_UpperCAmelCase									=  pt_model.visual.conva.weight.data
						_UpperCAmelCase									=  pt_model.visual.class_embedding
						_UpperCAmelCase									=  pt_model.visual.positional_embedding.data

						# copy encoder
						copy_layers(hf_model.vision_model.encoder.layers			,					pt_model.visual.transformer.resblocks		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  flatten_nested_dict(flax_params["backbone"]["merged_class_token"]		)

						_UpperCAmelCase									=  torch.from_numpy(flax_class_token_params["scale"]		)
						_UpperCAmelCase									=  torch.from_numpy(flax_class_token_params["bias"]		)
						_UpperCAmelCase									=  nn.Parameter(lowercase__		)
						_UpperCAmelCase									=  nn.Parameter(lowercase__		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  hf_model.state_dict()
						_UpperCAmelCase									=  {}

						# Rename class prediction head flax params to pytorch HF
						_UpperCAmelCase									=  flatten_nested_dict(flax_params["class_head"]		)

						for flax_key, v in flax_class_params.items():
												_UpperCAmelCase									=  flax_key.replace("/"			,					"."		)
												_UpperCAmelCase									=  torch_key.replace(".kernel"			,					".weight"		)
												_UpperCAmelCase									=  torch_key.replace("Dense_0"			,					"dense0"		)
												_UpperCAmelCase									=  "class_head." + torch_key

												if "weight" in torch_key and v.ndim == 2:
																		_UpperCAmelCase									=  v.T

												_UpperCAmelCase									=  nn.Parameter(torch.from_numpy(lowercase__		)		)

						# Rename box prediction box flax params to pytorch HF
						_UpperCAmelCase									=  flatten_nested_dict(flax_params["obj_box_head"]		)

						for flax_key, v in flax_box_params.items():
												_UpperCAmelCase									=  flax_key.replace("/"			,					"."		)
												_UpperCAmelCase									=  torch_key.replace(".kernel"			,					".weight"		)
												_UpperCAmelCase									=  torch_key.replace("_"			,					""		).lower()
												_UpperCAmelCase									=  "box_head." + torch_key

												if "weight" in torch_key and v.ndim == 2:
																		_UpperCAmelCase									=  v.T

												_UpperCAmelCase									=  nn.Parameter(torch.from_numpy(lowercase__		)		)

						# Copy flax params to PyTorch params
						for name, param in new_params.items():
												if name in pt_params.keys():
																		pt_params[name].copy_(lowercase__		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						for k, v in flax_attn_params.items():
												if k.startswith("transformer"		):
																		_UpperCAmelCase									=  k.replace("transformer.resblocks"			,					"text_model.encoder.layers"		)
												else:
																		_UpperCAmelCase									=  k.replace("visual.transformer.resblocks"			,					"vision_model.encoder.layers"		)

												_UpperCAmelCase									=  torch_key.replace("attn"			,					"self_attn"		)
												_UpperCAmelCase									=  torch_key.replace("key"			,					"k_proj"		)
												_UpperCAmelCase									=  torch_key.replace("value"			,					"v_proj"		)
												_UpperCAmelCase									=  torch_key.replace("query"			,					"q_proj"		)
												_UpperCAmelCase									=  torch_key.replace("out"			,					"out_proj"		)

												if "bias" in torch_key and v.ndim == 2:
																		_UpperCAmelCase									=  v.shape[0] * v.shape[1]
																		_UpperCAmelCase									=  v.reshape(lowercase__		)

												if "weight" in torch_key and "out" in torch_key:
																		_UpperCAmelCase									=  (v.shape[0] * v.shape[1], v.shape[2])
																		_UpperCAmelCase									=  v.reshape(lowercase__		).T

												if "weight" in torch_key and "out" not in torch_key:
																		_UpperCAmelCase									=  (v.shape[0], v.shape[1] * v.shape[2])
																		_UpperCAmelCase									=  v.reshape(lowercase__		).T

												# Copy flax CLIP attn params to HF PyTorch params
												_UpperCAmelCase									=  torch.from_numpy(lowercase__		)
												hf_backbone.state_dict()[torch_key].copy_(lowercase__		)


def 							__SCREAMING_SNAKE_CASE						(    snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  {}
						_UpperCAmelCase									=  []

						for k, v in params.items():
												if "attn." in k:
																		_UpperCAmelCase									=  k[: k.rindex("attn."		) + 5]
																		if base in processed_attn_layers:
																								continue

																		processed_attn_layers.append(lowercase__		)
																		_UpperCAmelCase									=  params[base + "out.weight"].shape[-1]
																		_UpperCAmelCase									=  params[base + "out.weight"].reshape(lowercase__			,					lowercase__		).T
																		_UpperCAmelCase									=  params[base + "out.bias"]
												else:
																		_UpperCAmelCase									=  v
						return new_params


def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  CLIP(**lowercase__		)
						torch_model.eval()
						_UpperCAmelCase									=  torch_model.state_dict()

						_UpperCAmelCase									=  flatten_nested_dict(flax_params["backbone"]["clip"]		)
						_UpperCAmelCase									=  {}

						for flax_key, v in flax_clip_params.items():
												_UpperCAmelCase									=  flax_key.replace("/"			,					"."		)
												_UpperCAmelCase									=  torch_key.replace("text.token_embedding.embedding"			,					"token_embedding.kernel"		)

												if (
												    torch_key.startswith("text.transformer"		)
												    or torch_key.startswith("text.text_projection"		)
												    or torch_key.startswith("text.ln_final"		)
												    or torch_key.startswith("text.positional_embedding"		)
												):
																		_UpperCAmelCase									=  torch_key[5:]

												_UpperCAmelCase									=  torch_key.replace("text_projection.kernel"			,					"text_projection"		)
												_UpperCAmelCase									=  torch_key.replace("visual.proj.kernel"			,					"visual.proj"		)
												_UpperCAmelCase									=  torch_key.replace(".scale"			,					".weight"		)
												_UpperCAmelCase									=  torch_key.replace(".kernel"			,					".weight"		)

												if "conv" in torch_key or "downsample.0.weight" in torch_key:
																		_UpperCAmelCase									=  v.transpose(3			,					2			,					0			,					1		)

												elif "weight" in torch_key and v.ndim == 2 and "embedding" not in torch_key:
																		# Fully connected layers are transposed, embeddings are not
																		_UpperCAmelCase									=  v.T

												_UpperCAmelCase									=  v

						_UpperCAmelCase									=  _convert_attn_layers(lowercase__		)
						new_torch_params.update(lowercase__		)
						_UpperCAmelCase									=  {}

						# Copy flax CLIP backbone params to PyTorch params
						for name, param in new_torch_params.items():
												if name in torch_clip_params.keys():
																		_UpperCAmelCase									=  torch.from_numpy(new_torch_params[name]		)
																		torch_clip_params[name].copy_(lowercase__		)
												else:
																		_UpperCAmelCase									=  param

						return torch_clip_params, torch_model, attn_params


@torch.no_grad()
def 							__SCREAMING_SNAKE_CASE						(    snake_case_			,					snake_case_			,					snake_case_			,					snake_case_			,					snake_case_=None		):

						'''simple docstring'''


						_UpperCAmelCase									=  Repository(lowercase__			,					clone_from=f"""google/{pytorch_dump_folder_path}"""		)
						repo.git_pull()

						if config_path is not None:
												_UpperCAmelCase									=  OwlViTConfig.from_pretrained(lowercase__		)
						else:
												_UpperCAmelCase									=  OwlViTConfig()

						_UpperCAmelCase									=  OwlViTModel(lowercase__		).eval()
						_UpperCAmelCase									=  OwlViTForObjectDetection(lowercase__		).eval()

						copy_text_model_and_projection(lowercase__			,					lowercase__		)
						copy_vision_model_and_projection(lowercase__			,					lowercase__		)
						_UpperCAmelCase									=  pt_backbone.logit_scale
						copy_flax_attn_params(lowercase__			,					lowercase__		)

						_UpperCAmelCase									=  hf_backbone
						copy_class_merge_token(lowercase__			,					lowercase__		)
						copy_class_box_heads(lowercase__			,					lowercase__		)

						# Save HF model
						hf_model.save_pretrained(repo.local_dir		)

						# Initialize image processor
						_UpperCAmelCase									=  OwlViTImageProcessor(
						    size=config.vision_config.image_size			,					crop_size=config.vision_config.image_size		)
						# Initialize tokenizer
						_UpperCAmelCase									=  CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32"			,					pad_token="!"			,					model_max_length=16		)

						# Initialize processor
						_UpperCAmelCase									=  OwlViTProcessor(image_processor=lowercase__			,					tokenizer=lowercase__		)
						image_processor.save_pretrained(repo.local_dir		)
						processor.save_pretrained(repo.local_dir		)

						repo.git_add()
						repo.git_commit("Upload model and processor"		)
						repo.git_push()


if __name__ == "__main__":
				lowercase_			:       Union[str, Any]						       =				argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--owlvit_version',
				    default=None,
				    type=str,
				    required=True,
				    help='OWL-ViT model name [clip_b16, clip_b32, clip_l14].',
				)
				parser.add_argument(
				    '--owlvit_checkpoint', default=None, type=str, required=True, help='Path to flax model checkpoint.'
				)
				parser.add_argument('--hf_config', default=None, type=str, required=True, help='Path to HF model config.')
				parser.add_argument(
				    '--pytorch_dump_folder_path', default='hf_model', type=str, help='Path to the output PyTorch model.'
				)
				lowercase_			:       Union[str, Any]						       =				parser.parse_args()

				# Initialize PyToch clip model
				lowercase_			:       Tuple						       =				args.owlvit_version
				if model_name == "clip_b16":
								lowercase_			:       Union[str, Any]						       =				CONFIGS['vit_b16']
				elif model_name == "clip_b32":
								lowercase_			:       int						       =				CONFIGS['vit_b32']
				elif model_name == "clip_l14":
								lowercase_			:       Optional[int]						       =				CONFIGS['vit_l14']

				# Load from checkpoint and convert params to float-32
				lowercase_			:       str						       =				checkpoints.restore_checkpoint(args.owlvit_checkpoint, target=None)['optimizer']['target']
				lowercase_			:       List[str]						       =				jax.tree_util.tree_map(lambda x: x.astype(jnp.floataa) if x.dtype == jnp.bfloataa else x, variables)
				del variables

				# Convert CLIP backbone
				lowercase_       ,  lowercase_       ,  lowercase_			:       Tuple						       =				convert_clip_backbone(flax_params, torch_config)

				convert_owlvit_checkpoint(clip_pt, flax_params, attn_params, args.pytorch_dump_folder_path, args.hf_config)










import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 					__lowerCAmelCase				( UpperCAmelCase__      ,	unittest.TestCase				):
			snake_case_   :				Optional[Any]				  =  ElectraTokenizer
			snake_case_   :				Dict				  =  ElectraTokenizerFast
			snake_case_   :				Optional[Any]				  =  True
			snake_case_   :				List[Any]				  =  True
			snake_case_   :				Optional[Any]				  =  filter_non_english







			def UpperCamelCase    ( self					:    Union[str, Any]   ):




									"""simple docstring"""


									super().setUp()

									_UpperCAmelCase									=  [
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "[PAD]",
									    "[MASK]",
									    "want",
									    "##want",
									    "##ed",
									    "wa",
									    "un",
									    "runn",
									    "##ing",
									    ",",
									    "low",
									    "lowest",
									]
									_UpperCAmelCase									=  os.path.join(self.tmpdirname					,					VOCAB_FILES_NAMES["vocab_file"]   )
									with open(self.vocab_file					,					"w"					,					encoding="utf-8"   ) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )







			def UpperCamelCase    ( self					:    Optional[Any]					,					snake_case__					:    Union[str, Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  "UNwant\u00E9d,running"
									_UpperCAmelCase									=  "unwanted, running"
									return input_text, output_text







			def UpperCamelCase    ( self					:    Optional[int]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self.tokenizer_class(self.vocab_file   )

									_UpperCAmelCase									=  tokenizer.tokenize("UNwant\u00E9d,running"   )
									self.assertListEqual(snake_case__					,					["un", "##want", "##ed", ",", "runn", "##ing"]   )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__   )					,					[9, 6, 7, 12, 10, 11]   )







			def UpperCamelCase    ( self					:    int   ):




									"""simple docstring"""


									if not self.test_rust_tokenizer:
															return

									_UpperCAmelCase									=  self.get_tokenizer()
									_UpperCAmelCase									=  self.get_rust_tokenizer()

									_UpperCAmelCase									=  "UNwant\u00E9d,running"

									_UpperCAmelCase									=  tokenizer.tokenize(snake_case__   )
									_UpperCAmelCase									=  rust_tokenizer.tokenize(snake_case__   )
									self.assertListEqual(snake_case__					,					snake_case__   )

									_UpperCAmelCase									=  tokenizer.encode(snake_case__					,					add_special_tokens=snake_case__   )
									_UpperCAmelCase									=  rust_tokenizer.encode(snake_case__					,					add_special_tokens=snake_case__   )
									self.assertListEqual(snake_case__					,					snake_case__   )

									_UpperCAmelCase									=  self.get_rust_tokenizer()
									_UpperCAmelCase									=  tokenizer.encode(snake_case__   )
									_UpperCAmelCase									=  rust_tokenizer.encode(snake_case__   )
									self.assertListEqual(snake_case__					,					snake_case__   )

									# With lower casing
									_UpperCAmelCase									=  self.get_tokenizer(do_lower_case=snake_case__   )
									_UpperCAmelCase									=  self.get_rust_tokenizer(do_lower_case=snake_case__   )

									_UpperCAmelCase									=  "UNwant\u00E9d,running"

									_UpperCAmelCase									=  tokenizer.tokenize(snake_case__   )
									_UpperCAmelCase									=  rust_tokenizer.tokenize(snake_case__   )
									self.assertListEqual(snake_case__					,					snake_case__   )

									_UpperCAmelCase									=  tokenizer.encode(snake_case__					,					add_special_tokens=snake_case__   )
									_UpperCAmelCase									=  rust_tokenizer.encode(snake_case__					,					add_special_tokens=snake_case__   )
									self.assertListEqual(snake_case__					,					snake_case__   )

									_UpperCAmelCase									=  self.get_rust_tokenizer()
									_UpperCAmelCase									=  tokenizer.encode(snake_case__   )
									_UpperCAmelCase									=  rust_tokenizer.encode(snake_case__   )
									self.assertListEqual(snake_case__					,					snake_case__   )







			def UpperCamelCase    ( self					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"   )					,					["ah", "\u535A", "\u63A8", "zz"]   )







			def UpperCamelCase    ( self					:    List[Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   )					,					["hello", "!", "how", "are", "you", "?"]   )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )					,					["hello"]   )







			def UpperCamelCase    ( self					:    Dict   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__					,					strip_accents=snake_case__   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )					,					["hällo", "!", "how", "are", "you", "?"]   )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )					,					["h\u00E9llo"]   )







			def UpperCamelCase    ( self					:    Optional[Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__					,					strip_accents=snake_case__   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )					,					["hallo", "!", "how", "are", "you", "?"]   )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )					,					["hello"]   )







			def UpperCamelCase    ( self					:    Union[str, Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )					,					["hallo", "!", "how", "are", "you", "?"]   )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )					,					["hello"]   )







			def UpperCamelCase    ( self					:    Union[str, Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   )					,					["HeLLo", "!", "how", "Are", "yoU", "?"]   )







			def UpperCamelCase    ( self					:    Dict   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__					,					strip_accents=snake_case__   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )					,					["HäLLo", "!", "how", "Are", "yoU", "?"]   )







			def UpperCamelCase    ( self					:    Optional[Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__					,					strip_accents=snake_case__   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )					,					["HaLLo", "!", "how", "Are", "yoU", "?"]   )







			def UpperCamelCase    ( self					:    List[str]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  BasicTokenizer(do_lower_case=snake_case__					,					never_split=["[UNK]"]   )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"   )					,					["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]   )







			def UpperCamelCase    ( self					:    Optional[int]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

									_UpperCAmelCase									=  {}
									for i, token in enumerate(snake_case__   ):
															_UpperCAmelCase									=  i
									_UpperCAmelCase									=  WordpieceTokenizer(vocab=snake_case__					,					unk_token="[UNK]"   )

									self.assertListEqual(tokenizer.tokenize(""   )					,					[]   )

									self.assertListEqual(tokenizer.tokenize("unwanted running"   )					,					["un", "##want", "##ed", "runn", "##ing"]   )

									self.assertListEqual(tokenizer.tokenize("unwantedX running"   )					,					["[UNK]", "runn", "##ing"]   )







			def UpperCamelCase    ( self					:    str   ):




									"""simple docstring"""


									self.assertTrue(_is_whitespace(" "   )   )
									self.assertTrue(_is_whitespace("\t"   )   )
									self.assertTrue(_is_whitespace("\r"   )   )
									self.assertTrue(_is_whitespace("\n"   )   )
									self.assertTrue(_is_whitespace("\u00A0"   )   )

									self.assertFalse(_is_whitespace("A"   )   )
									self.assertFalse(_is_whitespace("-"   )   )







			def UpperCamelCase    ( self					:    Optional[int]   ):




									"""simple docstring"""


									self.assertTrue(_is_control("\u0005"   )   )

									self.assertFalse(_is_control("A"   )   )
									self.assertFalse(_is_control(" "   )   )
									self.assertFalse(_is_control("\t"   )   )
									self.assertFalse(_is_control("\r"   )   )







			def UpperCamelCase    ( self					:    int   ):




									"""simple docstring"""


									self.assertTrue(_is_punctuation("-"   )   )
									self.assertTrue(_is_punctuation("$"   )   )
									self.assertTrue(_is_punctuation("`"   )   )
									self.assertTrue(_is_punctuation("."   )   )

									self.assertFalse(_is_punctuation("A"   )   )
									self.assertFalse(_is_punctuation(" "   )   )







			def UpperCamelCase    ( self					:    int   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self.get_tokenizer()
									_UpperCAmelCase									=  self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(snake_case__   ) for t in ["Test", "\xad", "test"]]					,					[["[UNK]"], [], ["[UNK]"]]   )

									self.assertListEqual(
									    [rust_tokenizer.tokenize(snake_case__   ) for t in ["Test", "\xad", "test"]]					,					[["[UNK]"], [], ["[UNK]"]]   )







			@slow
			def UpperCamelCase    ( self					:    str   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self.tokenizer_class.from_pretrained("google/electra-base-discriminator"   )

									_UpperCAmelCase									=  tokenizer.encode("sequence builders"					,					add_special_tokens=snake_case__   )
									_UpperCAmelCase									=  tokenizer.encode("multi-sequence build"					,					add_special_tokens=snake_case__   )

									_UpperCAmelCase									=  tokenizer.build_inputs_with_special_tokens(snake_case__   )
									_UpperCAmelCase									=  tokenizer.build_inputs_with_special_tokens(snake_case__					,					snake_case__   )

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]







			def UpperCamelCase    ( self					:    List[Any]   ):




									"""simple docstring"""


									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																					_UpperCAmelCase									=  self.rust_tokenizer_class.from_pretrained(snake_case__					,					**snake_case__   )

																					_UpperCAmelCase									=  F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																					_UpperCAmelCase									=  tokenizer_r.encode_plus(
																					    snake_case__					,					return_attention_mask=snake_case__					,					return_token_type_ids=snake_case__					,					return_offsets_mapping=snake_case__					,					add_special_tokens=snake_case__					,					)

																					_UpperCAmelCase									=  tokenizer_r.do_lower_case if hasattr(snake_case__					,					"do_lower_case"   ) else False
																					_UpperCAmelCase									=  (
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "A"),
																					        ((1, 2), ","),
																					        ((3, 5), "na"),
																					        ((5, 6), "##ï"),
																					        ((6, 8), "##ve"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "Allen"),
																					        ((21, 23), "##NL"),
																					        ((23, 24), "##P"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "a"),
																					        ((1, 2), ","),
																					        ((3, 8), "naive"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "allen"),
																					        ((21, 23), "##nl"),
																					        ((23, 24), "##p"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]					,					tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]   )   )
																					self.assertEqual([e[0] for e in expected_results]					,					tokens["offset_mapping"]   )







			def UpperCamelCase    ( self					:    Union[str, Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  ["的", "人", "有"]
									_UpperCAmelCase									=  "".join(snake_case__   )
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																					_UpperCAmelCase									=  True
																					_UpperCAmelCase									=  self.tokenizer_class.from_pretrained(snake_case__					,					**snake_case__   )
																					_UpperCAmelCase									=  self.rust_tokenizer_class.from_pretrained(snake_case__					,					**snake_case__   )

																					_UpperCAmelCase									=  tokenizer_p.encode(snake_case__					,					add_special_tokens=snake_case__   )
																					_UpperCAmelCase									=  tokenizer_r.encode(snake_case__					,					add_special_tokens=snake_case__   )

																					_UpperCAmelCase									=  tokenizer_r.convert_ids_to_tokens(snake_case__   )
																					_UpperCAmelCase									=  tokenizer_p.convert_ids_to_tokens(snake_case__   )

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(snake_case__					,					snake_case__   )
																					self.assertListEqual(snake_case__					,					snake_case__   )

																					_UpperCAmelCase									=  False
																					_UpperCAmelCase									=  self.rust_tokenizer_class.from_pretrained(snake_case__					,					**snake_case__   )
																					_UpperCAmelCase									=  self.tokenizer_class.from_pretrained(snake_case__					,					**snake_case__   )

																					_UpperCAmelCase									=  tokenizer_r.encode(snake_case__					,					add_special_tokens=snake_case__   )
																					_UpperCAmelCase									=  tokenizer_p.encode(snake_case__					,					add_special_tokens=snake_case__   )

																					_UpperCAmelCase									=  tokenizer_r.convert_ids_to_tokens(snake_case__   )
																					_UpperCAmelCase									=  tokenizer_p.convert_ids_to_tokens(snake_case__   )

																					# it is expected that only the first Chinese character is not preceded by "##".
																					_UpperCAmelCase									=  [
																					    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case__   )
																					]
																					self.assertListEqual(snake_case__					,					snake_case__   )
																					self.assertListEqual(snake_case__					,					snake_case__   )




