
import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand





class 						lowerCamelCase			(					__lowerCamelCase				):







	@staticmethod
	def    snake_case__    (    lowercase			:ArgumentParser )		->			Union[str, Any]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   parser.add_parser('''login'''      ,    help='''Log in using the same credentials as on huggingface.co''' )
				login_parser.set_defaults(func=lambda lowercase : LoginCommand(lowercase ) )
				SCREAMING_SNAKE_CASE    	=   parser.add_parser('''whoami'''      ,    help='''Find out which huggingface.co account you are logged in as.''' )
				whoami_parser.set_defaults(func=lambda lowercase : WhoamiCommand(lowercase ) )
				SCREAMING_SNAKE_CASE    	=   parser.add_parser('''logout'''      ,    help='''Log out''' )
				logout_parser.set_defaults(func=lambda lowercase : LogoutCommand(lowercase ) )

				# new system: git-based repo system
				SCREAMING_SNAKE_CASE    	=   parser.add_parser(
				    '''repo'''      ,    help='''Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.'''      ,    )
				SCREAMING_SNAKE_CASE    	=   repo_parser.add_subparsers(
				    help='''Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands''' )
				SCREAMING_SNAKE_CASE    	=   repo_subparsers.add_parser(
				    '''create'''      ,    help='''Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co''' )
				repo_create_parser.add_argument(
				    '''name'''      ,    type=lowercase      ,    help='''Name for your model\'s repo. Will be namespaced under your username to build the model id.'''      ,    )
				repo_create_parser.add_argument('''--organization'''      ,    type=lowercase      ,    help='''Optional: organization namespace.''' )
				repo_create_parser.add_argument('''-y'''      ,    '''--yes'''      ,    action='''store_true'''      ,    help='''Optional: answer Yes to the prompt''' )
				repo_create_parser.set_defaults(func=lambda lowercase : RepoCreateCommand(lowercase ) )





class 						lowerCamelCase			:
	UpperCamelCase_     :				List[str]								=  '\u001b[1m'
	UpperCamelCase_     :				Union[str, Any]								=  '\u001b[31m'
	UpperCamelCase_     :				Any								=  '\u001b[90m'
	UpperCamelCase_     :				List[str]								=  '\u001b[0m'







	@classmethod
	def    snake_case__    (    cls			:List[Any]      ,    lowercase			:Any )		->			List[str]:







				"""simple docstring"""






				return f"""{cls._bold}{s}{cls._reset}"""







	@classmethod
	def    snake_case__    (    cls			:Optional[Any]      ,    lowercase			:str )		->			Tuple:







				"""simple docstring"""






				return f"""{cls._bold}{cls._red}{s}{cls._reset}"""







	@classmethod
	def    snake_case__    (    cls			:List[str]      ,    lowercase			:Any )		->			List[Any]:







				"""simple docstring"""






				return f"""{cls._gray}{s}{cls._reset}"""



def       a			(   a		,	a  )    ->str:




			'''simple docstring'''


			SCREAMING_SNAKE_CASE    	=   [max(len(str(a  )  ) for x in col  ) for col in zip(*a		,	a  )]
			SCREAMING_SNAKE_CASE    	=   ('''{{:{}}} ''' * len(a  )).format(*a  )
			SCREAMING_SNAKE_CASE    	=   []
			lines.append(row_format.format(*a  )  )
			lines.append(row_format.format(*['''-''' * w for w in col_widths]  )  )
			for row in rows:
						lines.append(row_format.format(*a  )  )
			return "\n".join(a  )





class 						lowerCamelCase			:







	def __init__(    self			:List[str]      ,    lowercase			:Optional[Any] )		->			Union[str, Any]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   args





class 						lowerCamelCase			(					__lowerCamelCase				):







	def    snake_case__    (    self			:List[str] )		->			Optional[Any]:







				"""simple docstring"""






				print(
				    ANSI.red(
				        '''ERROR! `huggingface-cli login` uses an outdated login mechanism '''
				        '''that is not compatible with the Hugging Face Hub backend anymore. '''
				        '''Please use `huggingface-cli login instead.''' ) )





class 						lowerCamelCase			(					__lowerCamelCase				):







	def    snake_case__    (    self			:List[Any] )		->			Any:







				"""simple docstring"""






				print(
				    ANSI.red(
				        '''WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use '''
				        '''`huggingface-cli whoami` instead.''' ) )
				SCREAMING_SNAKE_CASE    	=   HfFolder.get_token()
				if token is None:
							print('''Not logged in''' )
							exit()
				try:
							SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   whoami(lowercase )
							print(lowercase )
							if orgs:
										print(ANSI.bold('''orgs: ''' )      ,    ''','''.join(lowercase ) )
				except HTTPError as e:
							print(lowercase )
							print(ANSI.red(e.response.text ) )
							exit(1 )





class 						lowerCamelCase			(					__lowerCamelCase				):







	def    snake_case__    (    self			:str )		->			Union[str, Any]:







				"""simple docstring"""






				print(
				    ANSI.red(
				        '''ERROR! `transformers-cli logout` uses an outdated logout mechanism '''
				        '''that is not compatible with the Hugging Face Hub backend anymore. '''
				        '''Please use `huggingface-cli logout instead.''' ) )





class 						lowerCamelCase			(					__lowerCamelCase				):







	def    snake_case__    (    self			:int )		->			Optional[Any]:







				"""simple docstring"""






				print(
				    ANSI.red(
				        '''WARNING! Managing repositories through transformers-cli is deprecated. '''
				        '''Please use `huggingface-cli` instead.''' ) )
				SCREAMING_SNAKE_CASE    	=   HfFolder.get_token()
				if token is None:
							print('''Not logged in''' )
							exit(1 )
				try:
							SCREAMING_SNAKE_CASE    	=   subprocess.check_output(['''git''', '''--version'''] ).decode('''utf-8''' )
							print(ANSI.gray(stdout.strip() ) )
				except FileNotFoundError:
							print('''Looks like you do not have git installed, please install.''' )

				try:
							SCREAMING_SNAKE_CASE    	=   subprocess.check_output(['''git-lfs''', '''--version'''] ).decode('''utf-8''' )
							print(ANSI.gray(stdout.strip() ) )
				except FileNotFoundError:
							print(
							    ANSI.red(
							        '''Looks like you do not have git-lfs installed, please install.'''
							        ''' You can install from https://git-lfs.github.com/.'''
							        ''' Then run `git lfs install` (you only have to do this once).''' ) )
				print('''''' )

				SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   whoami(lowercase )
				SCREAMING_SNAKE_CASE    	=   self.args.organization if self.args.organization is not None else user
				SCREAMING_SNAKE_CASE    	=   f"""{namespace}/{self.args.name}"""
				print(f"""You are about to create {ANSI.bold(lowercase )}""" )

				if not self.args.yes:
							SCREAMING_SNAKE_CASE    	=   input('''Proceed? [Y/n] ''' ).lower()
							if not (choice == "" or choice == "y" or choice == "yes"):
										print('''Abort''' )
										exit()
				try:
							SCREAMING_SNAKE_CASE    	=   create_repo(lowercase      ,    name=self.args.name      ,    organization=self.args.organization )
				except HTTPError as e:
							print(lowercase )
							print(ANSI.red(e.response.text ) )
							exit(1 )
				print('''\nYour repo now lives at:''' )
				print(f"""  {ANSI.bold(lowercase )}""" )
				print('''\nYou can clone it locally with the command below, and commit/push as usual.''' )
				print(f"""\n  git clone {url}""" )
				print('''''' )

import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 						lowerCamelCase			(					__lowerCamelCase   ,		unittest.TestCase				):
	UpperCamelCase_     :				List[str]								=  GPTNeoXJapaneseTokenizer
	UpperCamelCase_     :				List[Any]								=  False
	UpperCamelCase_     :				int								=  {'do_clean_text': False, 'add_prefix_space': False}







	def    snake_case__    (    self			:Optional[int] )		->			Union[str, Any]:







				"""simple docstring"""






				super().setUp()

				SCREAMING_SNAKE_CASE    	=   [
				    '''こん''',
				    '''こんに''',
				    '''にちは''',
				    '''ばんは''',
				    '''世界,㔺界''',
				    '''、''',
				    '''。''',
				    '''<BR>''',
				    '''<SP>''',
				    '''<TAB>''',
				    '''<URL>''',
				    '''<EMAIL>''',
				    '''<TEL>''',
				    '''<DATE>''',
				    '''<PRICE>''',
				    '''<BLOCK>''',
				    '''<KIGOU>''',
				    '''<U2000U2BFF>''',
				    '''<|emoji1|>''',
				    '''<unk>''',
				    '''<|startoftext|>''',
				    '''<|endoftext|>''',
				]
				SCREAMING_SNAKE_CASE    	=   {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
				SCREAMING_SNAKE_CASE    	=   {'''unk_token''': '''<unk>'''}

				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''vocab_file'''] )
				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''emoji_file'''] )
				with open(self.vocab_file      ,    '''w'''      ,    encoding='''utf-8''' ) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )
				with open(self.emoji_file      ,    '''w''' ) as emoji_writer:
							emoji_writer.write(json.dumps(lowercase ) )







	def    snake_case__    (    self			:Dict      ,    **lowercase			:Optional[int] )		->			int:







				"""simple docstring"""






				kwargs.update(self.special_tokens_map )
				return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,    **lowercase )







	def    snake_case__    (    self			:Dict      ,    lowercase			:Union[str, Any] )		->			List[str]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、世界。😀'''
				return input_text, output_text







	def    snake_case__    (    self			:Optional[int]      ,    lowercase			:Union[str, Any] )		->			List[str]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   self.get_input_output_texts(lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.encode(lowercase      ,    add_special_tokens=lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.decode(lowercase      ,    clean_up_tokenization_spaces=lowercase )
				return text, ids







	def    snake_case__    (    self			:Dict )		->			int:







				"""simple docstring"""






				pass  # TODO add if relevant







	def    snake_case__    (    self			:List[str] )		->			List[str]:







				"""simple docstring"""






				pass  # TODO add if relevant







	def    snake_case__    (    self			:Optional[Any] )		->			Union[str, Any]:







				"""simple docstring"""






				pass  # TODO add if relevant







	def    snake_case__    (    self			:Optional[Any] )		->			Dict:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   self.get_tokenizer()

				# Testing tokenization
				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。　こんばんは、㔺界。'''
				SCREAMING_SNAKE_CASE    	=   ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
				SCREAMING_SNAKE_CASE    	=   tokenizer.tokenize(lowercase )
				self.assertListEqual(lowercase      ,    lowercase )

				# Testing conversion to ids without special tokens
				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
				self.assertListEqual(lowercase      ,    lowercase )

				# Testing conversion to ids with special tokens
				SCREAMING_SNAKE_CASE    	=   tokens + [tokenizer.unk_token]
				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
				self.assertListEqual(lowercase      ,    lowercase )







	@slow
	def    snake_case__    (    self			:str )		->			Union[str, Any]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b''' )

				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''ありがとう。'''      ,    add_special_tokens=lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''どういたしまして。'''      ,    add_special_tokens=lowercase )

				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase      ,    lowercase )

				assert encoded_sentence == ids_a
				assert encoded_pair == ids_a + ids_a







	def    snake_case__    (    self			:List[Any] )		->			Dict:







				"""simple docstring"""






				pass







	def    snake_case__    (    self			:Any )		->			str:







				"""simple docstring"""






				pass