

from manim import *







class 		__magic_name__    (							lowerCamelCase__							):

   '''simple docstring'''




   def 				_UpperCAmelCase				(       self )							->					Dict:






     """simple docstring"""






     a__						=Rectangle(height=0.5,					width=0.5 )
     a__						=Rectangle(height=0.46,					width=0.46 ).set_stroke(width=0 )
     a__						=Rectangle(height=0.25,					width=0.25 )

     a__						=[mem.copy() for i in range(6 )]
     a__						=[mem.copy() for i in range(6 )]
     a__						=VGroup(*lowercase_ ).arrange(lowercase_,					buff=0 )
     a__						=VGroup(*lowercase_ ).arrange(lowercase_,					buff=0 )
     a__						=VGroup(lowercase_,					lowercase_ ).arrange(lowercase_,					buff=0 )
     a__						=Text('''CPU''',					font_size=24 )
     a__						=Group(lowercase_,					lowercase_ ).arrange(lowercase_,					buff=0.5,					aligned_edge=lowercase_ )
     cpu.move_to([-2.5, -0.5, 0] )
     self.add(lowercase_ )

     a__						=[mem.copy() for i in range(4 )]
     a__						=VGroup(*lowercase_ ).arrange(lowercase_,					buff=0 )
     a__						=Text('''GPU''',					font_size=24 )
     a__						=Group(lowercase_,					lowercase_ ).arrange(lowercase_,					buff=0.5,					aligned_edge=lowercase_ )
     gpu.move_to([-1, -1, 0] )
     self.add(lowercase_ )

     a__						=[mem.copy() for i in range(6 )]
     a__						=VGroup(*lowercase_ ).arrange(lowercase_,					buff=0 )

     a__						=Text('''Model''',					font_size=24 )
     a__						=Group(lowercase_,					lowercase_ ).arrange(lowercase_,					buff=0.5,					aligned_edge=lowercase_ )
     model.move_to([3, -1.0, 0] )
     self.add(lowercase_ )

     a__						=[]
     a__						=[]

     for i, rect in enumerate(lowercase_ ):
       rect.set_stroke(lowercase_ )

       a__						=Rectangle(height=0.46 / 4,					width=0.46 / 3 ).set_stroke(width=0.0 ).set_fill(lowercase_,					opacity=0.7 )

       if i == 0:
         cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT ),					buff=0.02,					direction=lowercase_ )
         cpu_target.set_x(cpu_target.get_x() + 0.1 )
       elif i == 3:
         cpu_target.next_to(model_cpu_arr[0],					direction=lowercase_,					buff=0.0 )
       else:
         cpu_target.next_to(model_cpu_arr[i - 1],					direction=lowercase_,					buff=0.0 )
       self.add(lowercase_ )
       model_cpu_arr.append(lowercase_ )

     self.add(*lowercase_,					*lowercase_ )

     a__						=[meta_mem.copy() for i in range(6 )]
     a__						=[meta_mem.copy() for i in range(6 )]
     a__						=VGroup(*lowercase_ ).arrange(lowercase_,					buff=0 )
     a__						=VGroup(*lowercase_ ).arrange(lowercase_,					buff=0 )
     a__						=VGroup(lowercase_,					lowercase_ ).arrange(lowercase_,					buff=0 )
     a__						=Text('''Disk''',					font_size=24 )
     a__						=Group(lowercase_,					lowercase_ ).arrange(lowercase_,					buff=0.5,					aligned_edge=lowercase_ )
     disk.move_to([-4.0, -1.25, 0] )
     self.add(lowercase_,					lowercase_ )

     a__						=[]

     for i in range(6 ):
       a__						=fill.copy().set_fill(lowercase_,					opacity=0.8 )
       target.move_to(disk_left_col_base[i] ).scale(0.5 )
       cpu_disk_arr.append(lowercase_ )

     self.add(*lowercase_ )

     a__						=Square(side_length=2.2 )
     key.move_to([-5, 2, 0] )

     a__						=MarkupText(
         F"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model""",					font_size=18,					)

     key_text.move_to([-5, 2.4, 0] )

     self.add(lowercase_,					lowercase_ )

     a__						=MarkupText(
         F"""<span fgcolor='{BLUE}'>●</span> Checkpoint""",					font_size=18,					)

     blue_text.next_to(lowercase_,					DOWN * 2.4,					aligned_edge=key_text.get_left() )
     self.add(lowercase_ )

     a__						=MarkupText(F"""The offloaded weights are all sent to the CPU.""",					font_size=24 )
     step_a.move_to([2, 2, 0] )

     self.play(Write(lowercase_,					run_time=3 ) )

     for i in range(6 ):
       a__						=cpu_disk_arr[i]
       a__						=rect.copy().set_fill(lowercase_,					opacity=0.8 ).scale(2.0 )
       cpa.generate_target()
       cpa.target.move_to(model_base[i] )

       if i == 0:
         rect.set_fill(lowercase_,					opacity=0.8 )
         rect.generate_target()
         rect.target.move_to(cpu_left_col_base[0] ).scale(2.0 )

         self.remove(
             *lowercase_,					*lowercase_,					)

       else:
         rect.generate_target()
         rect.target.move_to(cpu_left_col_base[i] ).scale(2.0 )
       self.play(MoveToTarget(lowercase_ ),					MoveToTarget(lowercase_ ),					model_base[i].animate.set_stroke(lowercase_ ) )
     self.play(FadeOut(lowercase_ ) )

     a__						=MarkupText(
         F"""Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.""",					font_size=24,					)
     step_a.move_to([2, 2, 0] )

     self.play(Write(lowercase_,					run_time=3 ) )

     a__						=[]
     a__						=[]
     for i in range(6 ):
       a__						=Arrow(start=lowercase_,					end=lowercase_,					color=lowercase_,					buff=0.5 )
       a.next_to(model_base[i].get_left(),					lowercase_,					buff=0.2 )
       arrows.append(lowercase_ )
       animations.append(Write(lowercase_ ) )
     self.play(*lowercase_ )
     self.wait()



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_xglm import XGLMTokenizer
else:
       lowerCamelCase     =      None


lowerCamelCase     =      logging.get_logger(__name__)

lowerCamelCase     =      {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase     =      {
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json''',
    },
}

lowerCamelCase     =      {
    '''facebook/xglm-564M''': 2048,
}







class 		__magic_name__    (							lowerCamelCase__							):

   '''simple docstring'''




   lowerCamelCase__				:			int												=     VOCAB_FILES_NAMES
   lowerCamelCase__				:			List[str]												=     PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				:			Dict												=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				:			Any												=     ['input_ids', 'attention_mask']
   lowerCamelCase__				:			Any												=     XGLMTokenizer



   def __init__(       self,					lowercase_=None,					lowercase_=None,					lowercase_="<s>",					lowercase_="</s>",					lowercase_="</s>",					lowercase_="<s>",					lowercase_="<unk>",					lowercase_="<pad>",					**lowercase_,					)							->					Any:






     """simple docstring"""






     a__						=7
     a__						=[F"""<madeupword{i}>""" for i in range(self.num_madeup_words )]

     a__						=kwargs.get('''additional_special_tokens''',					[] )
     kwargs["additional_special_tokens"] += [
         word for word in madeup_words if word not in kwargs["additional_special_tokens"]
     ]

     super().__init__(
         lowercase_,					tokenizer_file=lowercase_,					bos_token=lowercase_,					eos_token=lowercase_,					sep_token=lowercase_,					cls_token=lowercase_,					unk_token=lowercase_,					pad_token=lowercase_,					**lowercase_,					)

     a__						=vocab_file
     a__						=False if not self.vocab_file else True



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ = None )							->					List[int]:






     """simple docstring"""






     if token_ids_a is None:
       return [self.sep_token_id] + token_ids_a
     a__						=[self.sep_token_id]
     return sep + token_ids_a + sep + sep + token_ids_a



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ = None )							->					List[int]:






     """simple docstring"""






     a__						=[self.sep_token_id]

     if token_ids_a is None:
       return len(sep + token_ids_a ) * [0]
     return len(sep + token_ids_a + sep + sep + token_ids_a ) * [0]



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ = None )							->					Tuple[str]:






     """simple docstring"""






     if not self.can_save_slow_tokenizer:
       raise ValueError(
           '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
           '''tokenizer.''' )

     if not os.path.isdir(lowercase_ ):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory.""" )
       return
     a__						=os.path.join(
         lowercase_,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

     if os.path.abspath(self.vocab_file ) != os.path.abspath(lowercase_ ):
       copyfile(self.vocab_file,					lowercase_ )

     return (out_vocab_file,)
