



'''simple docstring'''



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowercase       :						List[Any]						=logging.get_logger(__name__)

_lowercase       :						Optional[Any]						={
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_lowercase       :						Tuple						={
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

_lowercase       :						Tuple						={
    "openai-gpt": 512,
}


def 							__UpperCAmelCase				(				UpperCamelCase__				:Union[str, Any]			)		->						Optional[int]:
				snake_case__      :   Optional[int]      =    text.strip()
				if not text:
								return []
				snake_case__      :   Union[str, Any]      =    text.split()
				return tokens






class 			_SCREAMING_SNAKE_CASE				(lowercase__      ):



			def __init__(     self		:						List[str]    ,			__UpperCamelCase		:						Union[str, Any]=True    ,			__UpperCamelCase		:						Optional[Any]=None    ,			__UpperCamelCase		:						Optional[int]=True    ,			__UpperCamelCase		:						Optional[Any]=None    ,			__UpperCamelCase		:						Dict=True    ,			)      ->      Any:
							"""simple docstring"""





							if never_split is None:
											snake_case__      :   int      =    []
							snake_case__      :   Optional[Any]      =    do_lower_case
							snake_case__      :   Dict      =    set(__UpperCamelCase			)
							snake_case__      :   List[str]      =    tokenize_chinese_chars
							snake_case__      :   Any      =    strip_accents
							snake_case__      :   Union[str, Any]      =    do_split_on_punc



			def 				lowerCAmelCase (     self		:						Any    ,			__UpperCamelCase		:						Any    ,			__UpperCamelCase		:						Any=None			)      ->      int:
							"""simple docstring"""





							snake_case__      :   int      =    self.never_split.union(set(__UpperCamelCase			)			) if never_split else self.never_split
							snake_case__      :   Optional[Any]      =    self._clean_text(__UpperCamelCase			)

							# This was added on November 1st, 2018 for the multilingual and Chinese
							# models. This is also applied to the English models now, but it doesn't
							# matter since the English models were not trained on any Chinese data
							# and generally don't have any Chinese data in them (there are Chinese
							# characters in the vocabulary because Wikipedia does have some Chinese
							# words in the English Wikipedia.).
							if self.tokenize_chinese_chars:
											snake_case__      :   Union[str, Any]      =    self._tokenize_chinese_chars(__UpperCamelCase			)
							# prevents treating the same character with different unicode codepoints as different characters
							snake_case__      :   str      =    unicodedata.normalize('''NFC'''    ,			__UpperCamelCase			)
							snake_case__      :   Any      =    whitespace_tokenize(__UpperCamelCase			)
							snake_case__      :   List[str]      =    []
							for token in orig_tokens:
											if token not in never_split:
															if self.do_lower_case:
																			snake_case__      :   Any      =    token.lower()
																			if self.strip_accents is not False:
																							snake_case__      :   List[Any]      =    self._run_strip_accents(__UpperCamelCase			)
															elif self.strip_accents:
																			snake_case__      :   Dict      =    self._run_strip_accents(__UpperCamelCase			)
											split_tokens.extend(self._run_split_on_punc(__UpperCamelCase    ,			__UpperCamelCase			)			)

							snake_case__      :   Dict      =    whitespace_tokenize(''' '''.join(__UpperCamelCase			)			)
							return output_tokens



			def 				lowerCAmelCase (     self		:						str    ,			__UpperCamelCase		:						List[Any]			)      ->      Union[str, Any]:
							"""simple docstring"""





							snake_case__      :   Dict      =    unicodedata.normalize('''NFD'''    ,			__UpperCamelCase			)
							snake_case__      :   Union[str, Any]      =    []
							for char in text:
											snake_case__      :   int      =    unicodedata.category(__UpperCamelCase			)
											if cat == "Mn":
															continue
											output.append(__UpperCamelCase			)
							return "".join(__UpperCamelCase			)



			def 				lowerCAmelCase (     self		:						str    ,			__UpperCamelCase		:						Optional[int]    ,			__UpperCamelCase		:						List[Any]=None			)      ->      Union[str, Any]:
							"""simple docstring"""





							if not self.do_split_on_punc or (never_split is not None and text in never_split):
											return [text]
							snake_case__      :   Optional[Any]      =    list(__UpperCamelCase			)
							snake_case__      :   str      =    0
							snake_case__      :   str      =    True
							snake_case__      :   Dict      =    []
							while i < len(__UpperCamelCase			):
											snake_case__      :   List[str]      =    chars[i]
											if _is_punctuation(__UpperCamelCase			):
															output.append([char]			)
															snake_case__      :   Optional[int]      =    True
											else:
															if start_new_word:
																			output.append([]			)
															snake_case__      :   Any      =    False
															output[-1].append(__UpperCamelCase			)
											i += 1

							return ["".join(__UpperCamelCase			) for x in output]



			def 				lowerCAmelCase (     self		:						Optional[Any]    ,			__UpperCamelCase		:						Tuple			)      ->      Optional[int]:
							"""simple docstring"""





							snake_case__      :   Optional[int]      =    []
							for char in text:
											snake_case__      :   str      =    ord(__UpperCamelCase			)
											if self._is_chinese_char(__UpperCamelCase			):
															output.append(''' '''			)
															output.append(__UpperCamelCase			)
															output.append(''' '''			)
											else:
															output.append(__UpperCamelCase			)
							return "".join(__UpperCamelCase			)



			def 				lowerCAmelCase (     self		:						Union[str, Any]    ,			__UpperCamelCase		:						Union[str, Any]			)      ->      str:
							"""simple docstring"""





							if (
							    (cp >= 0X4E_00 and cp <= 0X9F_FF)
							    or (cp >= 0X34_00 and cp <= 0X4D_BF)  #
							    or (cp >= 0X2_00_00 and cp <= 0X2_A6_DF)  #
							    or (cp >= 0X2_A7_00 and cp <= 0X2_B7_3F)  #
							    or (cp >= 0X2_B7_40 and cp <= 0X2_B8_1F)  #
							    or (cp >= 0X2_B8_20 and cp <= 0X2_CE_AF)  #
							    or (cp >= 0XF9_00 and cp <= 0XFA_FF)
							    or (cp >= 0X2_F8_00 and cp <= 0X2_FA_1F)  #
							):  #
											return True

							return False







			def 				lowerCAmelCase (     self		:						List[str]    ,			__UpperCamelCase		:						str			)      ->      Any:
							"""simple docstring"""





							snake_case__      :   str      =    []
							for char in text:
											snake_case__      :   List[Any]      =    ord(__UpperCamelCase			)
											if cp == 0 or cp == 0XFF_FD or _is_control(__UpperCamelCase			):
															continue
											if _is_whitespace(__UpperCamelCase			):
															output.append(''' '''			)
											else:
															output.append(__UpperCamelCase			)
							return "".join(__UpperCamelCase			)



def 							__UpperCAmelCase				(				UpperCamelCase__				:Optional[int]			)		->						Union[str, Any]:
				snake_case__      :   Dict      =    set()
				snake_case__      :   Tuple      =    word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)			)
								snake_case__      :   Dict      =    char
				return pairs


def 							__UpperCAmelCase				(				UpperCamelCase__				:Tuple			)		->						Dict:
				snake_case__      :   int      =    text.replace('''—'''   ,				'''-'''			)
				snake_case__      :   Optional[int]      =    text.replace('''–'''   ,				'''-'''			)
				snake_case__      :   Union[str, Any]      =    text.replace('''―'''   ,				'''-'''			)
				snake_case__      :   Any      =    text.replace('''…'''   ,				'''...'''			)
				snake_case__      :   Tuple      =    text.replace('''´'''   ,				'''\''''			)
				snake_case__      :   int      =    re.sub(R'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''   ,				R''' \1 '''   ,				UpperCamelCase__			)
				snake_case__      :   Optional[int]      =    re.sub(R'''\s*\n\s*'''   ,				''' \n '''   ,				UpperCamelCase__			)
				snake_case__      :   List[Any]      =    re.sub(R'''[^\S\n]+'''   ,				''' '''   ,				UpperCamelCase__			)
				return text.strip()






class 			_SCREAMING_SNAKE_CASE				(lowercase__      ):
			A__								=  VOCAB_FILES_NAMES
			A__								=  PRETRAINED_VOCAB_FILES_MAP
			A__								=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__								=  ['input_ids', 'attention_mask']



			def __init__(     self		:						Optional[Any]    ,			__UpperCamelCase		:						Any    ,			__UpperCamelCase		:						Any    ,			__UpperCamelCase		:						int="<unk>"    ,			**__UpperCamelCase		:						str			)      ->      Optional[Any]:
							"""simple docstring"""





							super().__init__(unk_token=__UpperCamelCase    ,			**__UpperCamelCase			)

							try:
											import ftfy
											from spacy.lang.en import English

											snake_case__      :   Union[str, Any]      =    English()
											snake_case__      :   List[str]      =    _nlp.tokenizer
											snake_case__      :   Tuple      =    ftfy.fix_text
							except ImportError:
											logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''			)
											snake_case__      :   List[Any]      =    BasicTokenizer(do_lower_case=__UpperCamelCase			)
											snake_case__      :   Optional[int]      =    None

							with open(__UpperCamelCase    ,			encoding='''utf-8'''			) as vocab_handle:
											snake_case__      :   Optional[Any]      =    json.load(__UpperCamelCase			)
							snake_case__      :   str      =    {v: k for k, v in self.encoder.items()}
							with open(__UpperCamelCase    ,			encoding='''utf-8'''			) as merges_handle:
											snake_case__      :   Optional[int]      =    merges_handle.read().split('''\n'''			)[1:-1]
							snake_case__      :   int      =    [tuple(merge.split()			) for merge in merges]
							snake_case__      :   Any      =    dict(zip(__UpperCamelCase    ,			range(len(__UpperCamelCase			)			)			)			)
							snake_case__      :   int      =    {}



			@property
			def 				lowerCAmelCase (     self		:						Dict			)      ->      Optional[int]:
							"""simple docstring"""





							return True



			@property
			def 				lowerCAmelCase (     self		:						str			)      ->      Any:
							"""simple docstring"""





							return len(self.encoder			)



			def 				lowerCAmelCase (     self		:						str			)      ->      Tuple:
							"""simple docstring"""





							return dict(self.encoder    ,			**self.added_tokens_encoder			)



			def 				lowerCAmelCase (     self		:						Dict    ,			__UpperCamelCase		:						Dict			)      ->      Union[str, Any]:
							"""simple docstring"""





							snake_case__      :   Optional[Any]      =    tuple(token[:-1]			) + (token[-1] + '''</w>''',)
							if token in self.cache:
											return self.cache[token]
							snake_case__      :   Union[str, Any]      =    get_pairs(__UpperCamelCase			)

							if not pairs:
											return token + "</w>"

							while True:
											snake_case__      :   Any      =    min(__UpperCamelCase    ,			key=lambda __UpperCamelCase			: self.bpe_ranks.get(__UpperCamelCase    ,			float('''inf'''			)			)			)
											if bigram not in self.bpe_ranks:
															break
											snake_case__  ,      snake_case__      :   Any      =    bigram
											snake_case__      :   List[Any]      =    []
											snake_case__      :   Any      =    0
											while i < len(__UpperCamelCase			):
															try:
																			snake_case__      :   Any      =    word.index(__UpperCamelCase    ,			__UpperCamelCase			)
															except ValueError:
																			new_word.extend(word[i:]			)
																			break
															else:
																			new_word.extend(word[i:j]			)
																			snake_case__      :   Any      =    j

															if word[i] == first and i < len(__UpperCamelCase			) - 1 and word[i + 1] == second:
																			new_word.append(first + second			)
																			i += 2
															else:
																			new_word.append(word[i]			)
																			i += 1
											snake_case__      :   Optional[Any]      =    tuple(__UpperCamelCase			)
											snake_case__      :   Dict      =    new_word
											if len(__UpperCamelCase			) == 1:
															break
											else:
															snake_case__      :   List[Any]      =    get_pairs(__UpperCamelCase			)
							snake_case__      :   Union[str, Any]      =    ''' '''.join(__UpperCamelCase			)
							if word == "\n  </w>":
											snake_case__      :   Union[str, Any]      =    '''\n</w>'''
							snake_case__      :   List[Any]      =    word
							return word



			def 				lowerCAmelCase (     self		:						Optional[Any]    ,			__UpperCamelCase		:						Optional[int]			)      ->      List[Any]:
							"""simple docstring"""





							snake_case__      :   Any      =    []
							if self.fix_text is None:
											# Using BERT's BasicTokenizer
											snake_case__      :   Any      =    self.nlp.tokenize(__UpperCamelCase			)
											for token in text:
															split_tokens.extend(list(self.bpe(__UpperCamelCase			).split(''' '''			)			)			)
							else:
											# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
											snake_case__      :   int      =    self.nlp(text_standardize(self.fix_text(__UpperCamelCase			)			)			)
											for token in text:
															split_tokens.extend(list(self.bpe(token.text.lower()			).split(''' '''			)			)			)
							return split_tokens



			def 				lowerCAmelCase (     self		:						Tuple    ,			__UpperCamelCase		:						Union[str, Any]			)      ->      List[Any]:
							"""simple docstring"""





							return self.encoder.get(__UpperCamelCase    ,			self.encoder.get(self.unk_token			)			)



			def 				lowerCAmelCase (     self		:						str    ,			__UpperCamelCase		:						Optional[Any]			)      ->      Optional[int]:
							"""simple docstring"""





							return self.decoder.get(__UpperCamelCase    ,			self.unk_token			)



			def 				lowerCAmelCase (     self		:						Optional[Any]    ,			__UpperCamelCase		:						int			)      ->      int:
							"""simple docstring"""





							snake_case__      :   Optional[int]      =    ''''''.join(__UpperCamelCase			).replace('''</w>'''    ,			''' '''			).strip()
							return out_string







			def 				lowerCAmelCase (     self		:						Optional[int]    ,			__UpperCamelCase		:						str    ,			__UpperCamelCase		:						Optional[str] = None			)      ->      Tuple[str]:
							"""simple docstring"""





							if not os.path.isdir(__UpperCamelCase			):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
											return
							snake_case__      :   Optional[Any]      =    os.path.join(
							    __UpperCamelCase    ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)
							snake_case__      :   str      =    os.path.join(
							    __UpperCamelCase    ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']			)

							with open(__UpperCamelCase    ,			'''w'''    ,			encoding='''utf-8'''			) as f:
											f.write(json.dumps(self.encoder    ,			indent=2    ,			sort_keys=__UpperCamelCase    ,			ensure_ascii=__UpperCamelCase			) + '''\n'''			)

							snake_case__      :   Union[str, Any]      =    0
							with open(__UpperCamelCase    ,			'''w'''    ,			encoding='''utf-8'''			) as writer:
											writer.write('''#version: 0.2\n'''			)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,			key=lambda __UpperCamelCase			: kv[1]			):
															if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																			    ''' Please check that the tokenizer is not corrupted!'''			)
																			snake_case__      :   Optional[int]      =    token_index
															writer.write(''' '''.join(__UpperCamelCase			) + '''\n'''			)
															index += 1

							return vocab_file, merge_file








'''simple docstring'''


import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only

from utils import save_json






def 			__lowerCAmelCase			(__lowerCAmelCase ):
		_UpperCAmelCase		:							Dict             =   filter(lambda __lowerCAmelCase : p.requires_grad   ,			model.parameters() )
		_UpperCAmelCase		:							List[Any]             =   sum([np.prod(p.size() ) for p in model_parameters] )
		return params


lowerCamelCase__     						=  logging.getLogger(__name__)






class 						lowerCAmelCase__      (      pl.Callback						):




			def 				lowerCAmelCase__     (		self	:  Optional[Any]						,	lowerCamelCase__	:  Optional[int]						,	lowerCamelCase__	:  Union[str, Any]			) ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase		:							int             =   {F"""lr_group_{i}""": param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups			)}
					pl_module.logger.log_metrics(lowerCamelCase__			)




			@rank_zero_only
			def 				lowerCAmelCase__     (		self	:  Dict						,	lowerCamelCase__	:  pl.Trainer						,	lowerCamelCase__	:  pl.LightningModule						,	lowerCamelCase__	:  str						,	lowerCamelCase__	:  Any=True			) ->None:



					'''simple docstring'''







					logger.info(F"""***** {type_path} results at step {trainer.global_step:05d} *****"""			)
					_UpperCAmelCase		:							Tuple             =   trainer.callback_metrics
					trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]}			)
					# Log results
					_UpperCAmelCase		:							Optional[int]             =   Path(pl_module.hparams.output_dir			)
					if type_path == "test":
							_UpperCAmelCase		:							Optional[Any]             =   od / "test_results.txt"
							_UpperCAmelCase		:							str             =   od / "test_generations.txt"
					else:
							# this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
							# If people want this it will be easy enough to add back.
							_UpperCAmelCase		:							Dict             =   od / F"""{type_path}_results/{trainer.global_step:05d}.txt"""
							_UpperCAmelCase		:							List[str]             =   od / F"""{type_path}_generations/{trainer.global_step:05d}.txt"""
							results_file.parent.mkdir(exist_ok=lowerCamelCase__			)
							generations_file.parent.mkdir(exist_ok=lowerCamelCase__			)
					with open(lowerCamelCase__						,	"a+"			) as writer:
							for key in sorted(lowerCamelCase__			):
									if key in ["log", "progress_bar", "preds"]:
											continue
									_UpperCAmelCase		:							Union[str, Any]             =   metrics[key]
									if isinstance(lowerCamelCase__						,	torch.Tensor			):
											_UpperCAmelCase		:							int             =   val.item()
									_UpperCAmelCase		:							List[str]             =   F"""{key}: {val:.6f}\n"""
									writer.write(lowerCamelCase__			)

					if not save_generations:
							return

					if "preds" in metrics:
							_UpperCAmelCase		:							Dict             =   "\n".join(metrics["preds"]			)
							generations_file.open("w+"			).write(lowerCamelCase__			)




			@rank_zero_only
			def 				lowerCAmelCase__     (		self	:  List[Any]						,	lowerCamelCase__	:  Dict						,	lowerCamelCase__	:  int			) ->Optional[int]:



					'''simple docstring'''







					try:
							_UpperCAmelCase		:							int             =   pl_module.model.model.num_parameters()
					except AttributeError:
							_UpperCAmelCase		:							Dict             =   pl_module.model.num_parameters()

					_UpperCAmelCase		:							Dict             =   count_trainable_parameters(lowerCamelCase__			)
					# mp stands for million parameters
					trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1E6, "grad_mp": n_trainable_pars / 1E6}			)




			@rank_zero_only
			def 				lowerCAmelCase__     (		self	:  List[str]						,	lowerCamelCase__	:  pl.Trainer						,	lowerCamelCase__	:  pl.LightningModule			) ->int:



					'''simple docstring'''







					save_json(pl_module.metrics						,	pl_module.metrics_save_path			)
					return self._write_logs(lowerCamelCase__						,	lowerCamelCase__						,	"test"			)






			@rank_zero_only
			def 				lowerCAmelCase__     (		self	:  List[str]						,	lowerCamelCase__	:  pl.Trainer						,	lowerCamelCase__	:  Tuple			) ->Dict:



					'''simple docstring'''







					save_json(pl_module.metrics						,	pl_module.metrics_save_path			)
					# Uncommenting this will save val generations
					# return self._write_logs(trainer, pl_module, "valid")






def 			__lowerCAmelCase			(__lowerCAmelCase   ,			__lowerCAmelCase   ,			__lowerCAmelCase=1   ,			__lowerCAmelCase=False ):
		if metric == "rouge2":
				_UpperCAmelCase		:							Tuple             =   "{val_avg_rouge2:.4f}-{step_count}"
		elif metric == "bleu":
				_UpperCAmelCase		:							List[str]             =   "{val_avg_bleu:.4f}-{step_count}"
		elif metric == "loss":
				_UpperCAmelCase		:							Union[str, Any]             =   "{val_avg_loss:.4f}-{step_count}"
		else:
				raise NotImplementedError(
				    F"""seq2seq callbacks only support rouge2, bleu and loss, got {metric}, You can make your own by adding to"""
				    " this function." )

		_UpperCAmelCase		:							Optional[int]             =   ModelCheckpoint(
		    dirpath=__lowerCAmelCase   ,			filename=__lowerCAmelCase   ,			monitor=F"""val_{metric}"""   ,			mode="min" if "loss" in metric else "max"   ,			save_top_k=__lowerCAmelCase   ,			)
		return checkpoint_callback






def 			__lowerCAmelCase			(__lowerCAmelCase   ,			__lowerCAmelCase ):
		return EarlyStopping(
		    monitor=F"""val_{metric}"""   ,			mode="min" if "loss" in metric else "max"   ,			patience=__lowerCAmelCase   ,			verbose=__lowerCAmelCase   ,			)



