






"""simple docstring"""


import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
   import jax
   import jax.numpy as jnp

   from transformers.modeling_flax_pytorch_utils import (
       convert_pytorch_state_dict_to_flax,
       load_flax_weights_in_pytorch_model,
   )
   from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
   import torch






class 		snake_case    :





    def __init__(				self      :   str							,						A      :   str							,						A      :   int=1_4							,						A      :   Optional[Any]=7							,						A      :   Optional[Any]=True							,						A      :   Optional[Any]=True							,						A      :   Optional[Any]=False							,						A      :   Any=True							,						A      :   Optional[int]=9_9							,						A      :   Tuple=3_2							,						A      :   Dict=5							,						A      :   Tuple=4							,						A      :   str=3_7							,						A      :   List[str]="gelu"							,						A      :   Optional[int]=0.1							,						A      :   str=0.1							,						A      :   List[Any]=5_1_2							,						A      :   int=0.02							,						):


      '''simple docstring'''




      a	:     str							=  parent
      a	:     List[str]							=  batch_size
      a	:     Optional[int]							=  seq_length
      a	:     Union[str, Any]							=  is_training
      a	:     str							=  use_input_mask
      a	:     str							=  use_token_type_ids
      a	:     List[str]							=  use_labels
      a	:     Dict							=  vocab_size
      a	:     Optional[int]							=  hidden_size
      a	:     Any							=  num_hidden_layers
      a	:     str							=  num_attention_heads
      a	:     List[Any]							=  intermediate_size
      a	:     Tuple							=  hidden_act
      a	:     int							=  hidden_dropout_prob
      a	:     Dict							=  attention_probs_dropout_prob
      a	:     Union[str, Any]							=  max_position_embeddings
      a	:     Any							=  initializer_range
      a	:     Optional[int]							=  None
      a	:     Optional[int]							=  vocab_size - 1
      a	:     Any							=  vocab_size - 1
      a	:     List[Any]							=  vocab_size - 1





    def 							lowerCamelCase__ (				self      :   Union[str, Any]     ):


      '''simple docstring'''




      a	:     Dict							=  ids_tensor([self.batch_size, self.seq_length]							,						self.vocab_size     )

      a	:     List[Any]							=  None
      if self.use_input_mask:
        a	:     Optional[Any]							=  random_attention_mask([self.batch_size, self.seq_length]     )

      a	:     Optional[Any]							=  GPTaConfig(
          vocab_size=self.vocab_size							,						n_embd=self.hidden_size							,						n_layer=self.num_hidden_layers							,						n_head=self.num_attention_heads							,						n_positions=self.max_position_embeddings							,						use_cache=A							,						bos_token_id=self.bos_token_id							,						eos_token_id=self.eos_token_id							,						pad_token_id=self.pad_token_id							,						)

      return (config, input_ids, input_mask)





    def 							lowerCamelCase__ (				self      :   List[Any]     ):


      '''simple docstring'''




      a	:     Any							=  self.prepare_config_and_inputs()
      a,	a,	a	:     int							=  config_and_inputs
      a	:     Optional[int]							=  {'input_ids': input_ids, 'attention_mask': attention_mask}
      return config, inputs_dict





    def 							lowerCamelCase__ (				self      :   Optional[int]     ):


      '''simple docstring'''




      a,	a,	a	:     List[str]							=  self.prepare_config_and_inputs()

      a	:     Tuple							=  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
      a	:     Union[str, Any]							=  ids_tensor([self.batch_size, self.seq_length]							,						vocab_size=2     )

      return (
          config,
          input_ids,
          attention_mask,
          encoder_hidden_states,
          encoder_attention_mask,
      )





    def 							lowerCamelCase__ (				self      :   List[str]							,						A      :   List[Any]							,						A      :   List[Any]							,						A      :   List[Any]							,						A      :   List[str]     ):


      '''simple docstring'''




      a	:     List[str]							=  2_0
      a	:     Optional[int]							=  model_class_name(A     )

      a	:     Optional[Any]							=  model.init_cache(input_ids.shape[0]							,						A     )
      a	:     Optional[int]							=  jnp.ones((input_ids.shape[0], max_decoder_length)							,						dtype='i4'     )

      a	:     Any							=  jnp.broadcast_to(
          jnp.arange(input_ids.shape[-1] - 1     )[None, :]							,						(input_ids.shape[0], input_ids.shape[-1] - 1)     )
      a	:     List[str]							=  model(
          input_ids[:, :-1]							,						attention_mask=A							,						past_key_values=A							,						position_ids=A							,						)

      a	:     Optional[int]							=  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]							,						dtype='i4'     )
      a	:     Dict							=  model(
          input_ids[:, -1:]							,						attention_mask=A							,						past_key_values=outputs_cache.past_key_values							,						position_ids=A							,						)

      a	:     List[Any]							=  model(A     )

      a	:     List[Any]							=  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])     )     )
      self.parent.assertTrue(diff < 1E-3							,						msg=F'''Max diff is {diff}'''     )





    def 							lowerCamelCase__ (				self      :   Tuple							,						A      :   Tuple							,						A      :   Tuple							,						A      :   Tuple							,						A      :   Union[str, Any]     ):


      '''simple docstring'''




      a	:     Tuple							=  2_0
      a	:     Union[str, Any]							=  model_class_name(A     )

      a	:     str							=  jnp.concatenate(
          [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])     )]							,						axis=-1							,						)

      a	:     List[str]							=  model.init_cache(input_ids.shape[0]							,						A     )
      a	:     str							=  jnp.broadcast_to(
          jnp.arange(input_ids.shape[-1] - 1     )[None, :]							,						(input_ids.shape[0], input_ids.shape[-1] - 1)     )

      a	:     int							=  model(
          input_ids[:, :-1]							,						attention_mask=A							,						past_key_values=A							,						position_ids=A							,						)
      a	:     Tuple							=  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]							,						dtype='i4'     )
      a	:     List[str]							=  model(
          input_ids[:, -1:]							,						past_key_values=outputs_cache.past_key_values							,						attention_mask=A							,						position_ids=A							,						)

      a	:     Dict							=  model(A							,						attention_mask=A     )

      a	:     Union[str, Any]							=  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])     )     )
      self.parent.assertTrue(diff < 1E-3							,						msg=F'''Max diff is {diff}'''     )






@require_flax
class 		snake_case    (    UpperCAmelCase  ,					UpperCAmelCase  ,					unittest.TestCase			):
    __magic_name__       	=	(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
    __magic_name__       	=	(FlaxGPTaLMHeadModel,) if is_flax_available() else ()





    def 							lowerCamelCase__ (				self      :   List[Any]     ):


      '''simple docstring'''




      a	:     int							=  FlaxGPTaModelTester(self     )





    def 							lowerCamelCase__ (				self      :   Union[str, Any]     ):


      '''simple docstring'''




      for model_class_name in self.all_model_classes:
        a,	a,	a	:     Tuple							=  self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_use_cache_forward(A							,						A							,						A							,						A     )





    def 							lowerCamelCase__ (				self      :   str     ):


      '''simple docstring'''




      for model_class_name in self.all_model_classes:
        a,	a,	a	:     List[str]							=  self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_use_cache_forward_with_attn_mask(
            A							,						A							,						A							,						A     )





    @slow
    def 							lowerCamelCase__ (				self      :   str     ):


      '''simple docstring'''




      a	:     List[Any]							=  GPTaTokenizer.from_pretrained('gpt2'							,						pad_token='</s>'							,						padding_side='left'     )
      a	:     int							=  tokenizer(['Hello this is a long string', 'Hey']							,						return_tensors='np'							,						padding=A							,						truncation=A     )

      a	:     Optional[Any]							=  FlaxGPTaLMHeadModel.from_pretrained('gpt2'     )
      a	:     Tuple							=  False
      a	:     List[Any]							=  model.config.eos_token_id

      a	:     List[Any]							=  jax.jit(model.generate     )

      a	:     Union[str, Any]							=  jit_generate(inputs['input_ids']							,						attention_mask=inputs['attention_mask']     ).sequences

      a	:     Union[str, Any]							=  tokenizer.batch_decode(A							,						skip_special_tokens=A     )

      a	:     List[str]							=  [
          'Hello this is a long string of words. I\'m going to start with the first one.\n',
          'Hey, I\'m not sure if I\'m going to be able to do',
      ]

      self.assertListEqual(A							,						A     )





    @is_pt_flax_cross_test
    def 							lowerCamelCase__ (				self      :   Optional[Any]     ):


      '''simple docstring'''




      a,	a	:     Tuple							=  self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
        with self.subTest(model_class.__name__     ):
          # prepare inputs
          a	:     Tuple							=  self._prepare_for_class(A							,						A     )
          a	:     int							=  {k: torch.tensor(v.tolist()     ) for k, v in prepared_inputs_dict.items()}

          # load corresponding PyTorch class
          a	:     Dict							=  model_class.__name__[4:]  # Skip the "Flax" at the beginning
          a	:     Optional[int]							=  getattr(A							,						A     )

          a,	a	:     Tuple							=  pt_inputs['input_ids'].shape
          a	:     Optional[Any]							=  np.random.randint(0							,						seq_length - 1							,						size=(batch_size,)     )
          for batch_idx, start_index in enumerate(A     ):
            a	:     Union[str, Any]							=  0
            a	:     List[Any]							=  1
            a	:     Union[str, Any]							=  0
            a	:     List[str]							=  1
          a	:     str							=  pt_model_class(A     ).eval()
          a	:     int							=  model_class(A							,						dtype=jnp.floataa     )

          a	:     List[str]							=  convert_pytorch_state_dict_to_flax(pt_model.state_dict()							,						A     )
          a	:     Tuple							=  fx_state

          with torch.no_grad():
            a	:     int							=  pt_model(**A     ).to_tuple()

          a	:     Any							=  fx_model(**A     ).to_tuple()
          self.assertEqual(len(A     )							,						len(A     )							,						'Output lengths differ between Flax and PyTorch'     )
          for fx_output, pt_output in zip(A							,						A     ):
            self.assert_almost_equals(fx_output[:, -1]							,						pt_output[:, -1].numpy()							,						4E-2     )

          with tempfile.TemporaryDirectory() as tmpdirname:
            pt_model.save_pretrained(A     )
            a	:     Union[str, Any]							=  model_class.from_pretrained(A							,						from_pt=A     )

          a	:     List[Any]							=  fx_model_loaded(**A     ).to_tuple()
          self.assertEqual(
              len(A     )							,						len(A     )							,						'Output lengths differ between Flax and PyTorch'     )
          for fx_output_loaded, pt_output in zip(A							,						A     ):
            self.assert_almost_equals(fx_output_loaded[:, -1]							,						pt_output[:, -1].numpy()							,						4E-2     )





    @is_pt_flax_cross_test
    def 							lowerCamelCase__ (				self      :   Optional[Any]     ):


      '''simple docstring'''




      a,	a	:     Optional[int]							=  self.model_tester.prepare_config_and_inputs_for_common()
      for model_class in self.all_model_classes:
        with self.subTest(model_class.__name__     ):
          # prepare inputs
          a	:     Tuple							=  self._prepare_for_class(A							,						A     )
          a	:     int							=  {k: torch.tensor(v.tolist()     ) for k, v in prepared_inputs_dict.items()}

          # load corresponding PyTorch class
          a	:     List[Any]							=  model_class.__name__[4:]  # Skip the "Flax" at the beginning
          a	:     List[str]							=  getattr(A							,						A     )

          a	:     Any							=  pt_model_class(A     ).eval()
          a	:     Any							=  model_class(A							,						dtype=jnp.floataa     )

          a	:     List[Any]							=  load_flax_weights_in_pytorch_model(A							,						fx_model.params     )
          a,	a	:     int							=  pt_inputs['input_ids'].shape
          a	:     Union[str, Any]							=  np.random.randint(0							,						seq_length - 1							,						size=(batch_size,)     )
          for batch_idx, start_index in enumerate(A     ):
            a	:     Tuple							=  0
            a	:     List[str]							=  1
            a	:     Any							=  0
            a	:     Dict							=  1

          # make sure weights are tied in PyTorch
          pt_model.tie_weights()

          with torch.no_grad():
            a	:     List[str]							=  pt_model(**A     ).to_tuple()

          a	:     Tuple							=  fx_model(**A     ).to_tuple()
          self.assertEqual(len(A     )							,						len(A     )							,						'Output lengths differ between Flax and PyTorch'     )
          for fx_output, pt_output in zip(A							,						A     ):
            self.assert_almost_equals(fx_output[:, -1]							,						pt_output[:, -1].numpy()							,						4E-2     )

          with tempfile.TemporaryDirectory() as tmpdirname:
            fx_model.save_pretrained(A     )
            a	:     Tuple							=  pt_model_class.from_pretrained(A							,						from_flax=A     )

          with torch.no_grad():
            a	:     Any							=  pt_model_loaded(**A     ).to_tuple()

          self.assertEqual(
              len(A     )							,						len(A     )							,						'Output lengths differ between Flax and PyTorch'     )
          for fx_output, pt_output in zip(A							,						A     ):
            self.assert_almost_equals(fx_output[:, -1]							,						pt_output[:, -1].numpy()							,						4E-2     )





    @slow
    def 							lowerCamelCase__ (				self      :   Tuple     ):


      '''simple docstring'''




      for model_class_name in self.all_model_classes:
        a	:     Union[str, Any]							=  model_class_name.from_pretrained('gpt2'							,						from_pt=A     )
        a	:     List[str]							=  model(np.ones((1, 1)     )     )
        self.assertIsNotNone(A     )












"""simple docstring"""


_UpperCamelCase							:       str       					=		10**8




def      snake_case    (A_ :int ):







  '''simple docstring'''

  if n % 1_0 == 0:
    return False
  a	:     Optional[int]							=  str(A_ )
  return s == s[::-1]




def      snake_case    ():







  '''simple docstring'''

  a	:     Tuple							=  set()
  a	:     Optional[Any]							=  1
  a	:     List[str]							=  5
  while sum_squares < LIMIT:
    a	:     int							=  first_square + 1
    while sum_squares < LIMIT:
      if is_palindrome(A_ ):
        answer.add(A_ )
      last_square += 1
      sum_squares += last_square**2
    first_square += 1
    a	:     List[str]							=  first_square**2 + (first_square + 1) ** 2

  return sum(A_ )


if __name__ == "__main__":
   print(solution())




