

"""simple docstring"""





import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
     import importlib.resources as pkg_resources
except ImportError:
     # Try backported to PY<37 `importlib_resources`.
     import importlib_resources as pkg_resources

from . import resources


_a				   =							'https://github.com/huggingface/datasets/tree/main/src/datasets/utils'
_a				   =							F"{BASE_REF_URL}/{__file__}"
_a				   =							logging.getLogger(__name__)




def 						_A						(						UpperCamelCase_  : str)			->		Tuple[Any, str]:
   '''simple docstring'''


   __lowercase			=		pkg_resources.read_text(UpperCamelCase_,    UpperCamelCase_)
   return yaml.safe_load(UpperCamelCase_), F"""{BASE_REF_URL}/resources/{resource}"""


_a   ,		_a				   =							load_yaml_resource('readme_structure.yaml')

_a				   =							[
    '[Needs More Information]',
    '[More Information Needed]',
    '(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)',
]

# Dictionary representation of section/readme, error_list, warning_list
_a				   =							Tuple[dict, List[str], List[str]]




class       _lowerCAmelCase						:







 """simple docstring"""







 def __init__(    self							:    Union[str, Any],      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    List[str] = None,      UpperCAmelCase__							:    bool = False			):
    __lowercase			=		name
    __lowercase			=		level
    __lowercase			=		lines
    __lowercase			=		""
    __lowercase			=		True
    __lowercase			=		{}
    __lowercase			=		[]
    __lowercase			=		[]
    if self.lines is not None:
       self.parse(suppress_parsing_errors=UpperCAmelCase__			)


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    bool = False			):
    __lowercase			=		""
    __lowercase			=		[]
    __lowercase			=		False
    for line in self.lines:
       if line.strip(" \n"			) == "":
          continue
       elif line.strip(" \n"			)[:3] == "```":
          __lowercase			=		not code_start
       elif line.split()[0] == self.level + "#" and not code_start:
          if current_sub_level != "":
             __lowercase			=		Section(UpperCAmelCase__,      self.level + "#",      UpperCAmelCase__			)
             __lowercase			=		[]
          else:
             if current_lines != []:
                self.text += "".join(UpperCAmelCase__			).strip()
                if self.text != "" and self.text not in FILLER_TEXT:
                   __lowercase			=		False
                __lowercase			=		[]

          __lowercase			=		" ".join(line.split()[1:]			).strip(" \n"			)
       else:
          current_lines.append(UpperCAmelCase__			)
    else:
       if current_sub_level != "":
          if current_sub_level in self.content:
             self.parsing_error_list.append(
                 F"""Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections."""			)
          __lowercase			=		Section(UpperCAmelCase__,      self.level + "#",      UpperCAmelCase__			)
       else:
          if current_lines != []:
             self.text += "".join(UpperCAmelCase__			).strip()
             if self.text != "" and self.text not in FILLER_TEXT:
                __lowercase			=		False

    if self.level == "" and not suppress_parsing_errors:
       if self.parsing_error_list != [] or self.parsing_warning_list != []:
          __lowercase			=		__lowercase			=		"\n".join("-\t" + x for x in self.parsing_error_list + self.parsing_warning_list			)
          __lowercase			=		F"""The following issues were found while parsing the README at `{self.name}`:\n""" + errors
          raise ValueError(UpperCAmelCase__			)


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    dict			):
    __lowercase			=		[]
    __lowercase			=		[]
    if structure["allow_empty"] is False:
       # If content is expected
       if self.is_empty_text and self.content == {}:
          # If no content is found, mention it in the error_list
          error_list.append(F"""Expected some content in section `{self.name}` but it is empty."""			)

    if structure["allow_empty_text"] is False:
       # If some text is expected
       if self.is_empty_text:
          # If no text is found, mention it in the error_list
          error_list.append(
              F"""Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)."""			)
        # Subsections Validation
    if structure["subsections"] is not None:
       # If subsections are expected
       if self.content == {}:
          # If no subsections are present
          __lowercase			=		[subsection["name"] for subsection in structure["subsections"]]
          # Mention the expected values in the error_list
          error_list.append(
              F"""Section `{self.name}` expected the following subsections: {", ".join(["`"+x+"`" for x in values]			)}. Found 'None'."""			)
       else:
          # If some subsections are present
          __lowercase			=		[subsection["name"] for subsection in structure["subsections"]]
          __lowercase			=		False
          for idx, name in enumerate(UpperCAmelCase__			):
             if name not in self.content:
                # If the expected subsection is not present
                error_list.append(F"""Section `{self.name}` is missing subsection: `{name}`."""			)
                __lowercase			=		True
             else:
                # If the subsection is present, validate subsection, return the result
                # and concat the errors from subsection to section error_list

                # Skip sublevel validation if current level is `###`
                if self.level == "###":
                   continue
                else:
                   __lowercase	,__lowercase	,__lowercase			=		self.content[name].validate(
                       structure["subsections"][idx]			)
                error_list += subsec_error_list
                warning_list += subsec_warning_list

          if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
             for name in self.content:
                if name not in structure_names:
                   # If an extra subsection is present
                   warning_list.append(
                       F"""`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown."""			)
    if error_list:
       # If there are errors, do not return the dictionary as it is invalid
       return {}, error_list, warning_list
    else:
       return self.to_dict(), error_list, warning_list



 def       _lowercase				(    self							:    Dict			):
    return {
        "name": self.name,
        "text": self.text,
        "is_empty_text": self.is_empty_text,
        "subsections": [value.to_dict() for value in self.content.values()],
    }







class       _lowerCAmelCase						(			lowercase     ):  # Level 0







 """simple docstring"""







 def __init__(    self							:    Tuple,      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    List[str],      UpperCAmelCase__							:    dict = None,      UpperCAmelCase__							:    bool = False			):
    super().__init__(name=UpperCAmelCase__,      level=""			)  # Not using lines here as we need to use a child class parse
    __lowercase			=		structure
    __lowercase			=		-2
    __lowercase			=		0
    __lowercase			=		lines
    if self.lines is not None:
       self.parse(suppress_parsing_errors=UpperCAmelCase__			)


 def       _lowercase				(    self							:    int			):
    if self.structure is None:
       __lowercase	,__lowercase	,__lowercase			=		self._validate(UpperCAmelCase__			)
    else:
       __lowercase	,__lowercase	,__lowercase			=		self._validate(self.structure			)
    if error_list != [] or warning_list != []:
       __lowercase			=		"\n".join(["-\t" + x for x in error_list + warning_list]			)
       __lowercase			=		F"""The following issues were found for the README at `{self.name}`:\n""" + errors
       raise ValueError(UpperCAmelCase__			)


 @classmethod
 def       _lowercase				(    cls							:    Tuple,      UpperCAmelCase__							:    Path,      UpperCAmelCase__							:    dict = None,      UpperCAmelCase__							:    bool = False			):
    with open(UpperCAmelCase__,      encoding="utf-8"			) as f:
       __lowercase			=		f.readlines()
    return cls(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      suppress_parsing_errors=UpperCAmelCase__			)


 @classmethod
 def       _lowercase				(    cls							:    Dict,      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    dict = None,      UpperCAmelCase__							:    str = "root",      UpperCAmelCase__							:    bool = False			):
    __lowercase			=		string.split("\n"			)
    return cls(UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      suppress_parsing_errors=UpperCAmelCase__			)


 def       _lowercase				(    self							:    str,      UpperCAmelCase__							:    bool = False			):
    # Skip Tags
    __lowercase			=		0

    for line in self.lines:
       self.yaml_tags_line_count += 1
       if line.strip(" \n"			) == "---":
          self.tag_count += 1
          if self.tag_count == 2:
             break
       line_count += 1
    if self.tag_count == 2:
       __lowercase			=		self.lines[line_count + 1 :]  # Get the last + 1 th item.
    else:
       __lowercase			=		self.lines[self.tag_count :]
    super().parse(suppress_parsing_errors=UpperCAmelCase__			)


 def __str__(    self							:    List[Any]			):
    return str(self.to_dict()			)



 def       _lowercase				(    self							:    Tuple,      UpperCAmelCase__							:    List[Any]			):
    __lowercase			=		[]
    __lowercase			=		[]
    if self.yaml_tags_line_count == 0:
       warning_list.append("Empty YAML markers are present in the README."			)
    elif self.tag_count == 0:
       warning_list.append("No YAML markers are present in the README."			)
    elif self.tag_count == 1:
       warning_list.append("Only the start of YAML tags present in the README."			)
    # Check how many first level sections are present.
    __lowercase			=		len(self.content.keys()			)
    if num_first_level_keys > 1:
       # If more than one, add to the error list, continue
       error_list.append(
           F"""The README has several first-level headings: {", ".join(["`"+x+"`" for x in list(self.content.keys()			)]			)}. Only one heading is expected. Skipping further validation for this README."""			)
    elif num_first_level_keys < 1:
       # If less than one, append error.
       error_list.append(
           "The README has no first-level headings. One heading is expected. Skipping further validation for this README."			)

    else:
       # If one exactly
       __lowercase			=		list(self.content.keys()			)[0]  # Get the key
       if start_key.startswith("Dataset Card for"			):  # Check correct start
          # If the starting is correct, validate all the sections
          __lowercase	,__lowercase	,__lowercase			=		self.content[start_key].validate(
              readme_structure["subsections"][0]			)
          error_list += sec_error_list
          warning_list += sec_warning_list
       else:
          # If not found, append error
          error_list.append(
              "No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."			)
    if error_list:
       # If there are errors, do not return the dictionary as it is invalid
       return {}, error_list, warning_list
    else:
       return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
     from argparse import ArgumentParser

     _a				   =							ArgumentParser(usage='Validate the content (excluding YAML tags) of a README.md file.')
     ap.add_argument('readme_filepath')
     _a				   =							ap.parse_args()
     _a				   =							Path(args.readme_filepath)
     _a				   =							ReadMe.from_readme(readme_filepath)







"""simple docstring"""





import argparse
import json
import os.path
from collections import OrderedDict

import numpy as np
import requests
import torch
from flax.training.checkpoints import restore_checkpoint
from huggingface_hub import hf_hub_download

from transformers import VivitConfig, VivitForVideoClassification, VivitImageProcessor
from transformers.image_utils import PILImageResampling




def 						_A						(						UpperCamelCase_  : str)			->		Optional[Any]:
   '''simple docstring'''


   __lowercase			=		"https://storage.googleapis.com/scenic-bucket/vivit/kinetics_400/vivit_base_16x2_unfactorized/checkpoint"

   with open(UpperCamelCase_,    "wb") as f:
      with requests.get(UpperCamelCase_,    stream=UpperCamelCase_) as req:
         for chunk in req.iter_content(chunk_size=2048):
            f.write(UpperCamelCase_)




def 						_A						(						)			->		VivitConfig:
   '''simple docstring'''


   __lowercase			=		VivitConfig()

   __lowercase			=		400
   __lowercase			=		"huggingface/label-files"
   __lowercase			=		"kinetics400-id2label.json"

   __lowercase			=		json.load(open(hf_hub_download(UpperCamelCase_,    UpperCamelCase_,    repo_type="dataset"),    "r"))
   __lowercase			=		{int(UpperCamelCase_): v for k, v in idalabel.items()}
   __lowercase			=		idalabel
   __lowercase			=		{v: k for k, v in idalabel.items()}
   return config




def 						_A						(						)			->		List[Any]:
   '''simple docstring'''


   __lowercase			=		hf_hub_download(
       repo_id="hf-internal-testing/spaghetti-video",    filename="eating_spaghetti_32_frames.npy",    repo_type="dataset")
   __lowercase			=		np.load(UpperCamelCase_)
   return list(UpperCamelCase_)




def 						_A						(						UpperCamelCase_  : np.ndarray)			->		Optional[int]:
   '''simple docstring'''


   if np.ndim(UpperCamelCase_) == 2:
      return transform_attention_bias(UpperCamelCase_)

   elif np.ndim(UpperCamelCase_) == 3:
      return transform_attention_kernel(UpperCamelCase_)

   else:
      raise Exception(F"""Invalid number of dimesions: {np.ndim(UpperCamelCase_)}""")




def 						_A						(						UpperCamelCase_  : np.ndarray)			->		Optional[Any]:
   '''simple docstring'''


   return current.flatten()




def 						_A						(						UpperCamelCase_  : np.ndarray)			->		Any:
   '''simple docstring'''


   return np.reshape(UpperCamelCase_,    (current.shape[0], current.shape[1] * current.shape[2])).T




def 						_A						(						UpperCamelCase_  : np.ndarray)			->		Any:
   '''simple docstring'''


   return np.reshape(UpperCamelCase_,    (current.shape[0] * current.shape[1], current.shape[2])).T




def 						_A						(						UpperCamelCase_  : Tuple,    UpperCamelCase_  : Optional[Any])			->		Any:
   '''simple docstring'''


   __lowercase			=		state_dict["optimizer"]["target"]["Transformer"][F"""encoderblock_{i}"""]

   __lowercase			=		F"""encoder.layer.{i}."""
   __lowercase			=		{
       prefix + "intermediate.dense.bias": state["MlpBlock_0"]["Dense_0"]["bias"],
       prefix + "intermediate.dense.weight": np.transpose(state["MlpBlock_0"]["Dense_0"]["kernel"]),
       prefix + "output.dense.bias": state["MlpBlock_0"]["Dense_1"]["bias"],
       prefix + "output.dense.weight": np.transpose(state["MlpBlock_0"]["Dense_1"]["kernel"]),
       prefix + "layernorm_before.bias": state["LayerNorm_0"]["bias"],
       prefix + "layernorm_before.weight": state["LayerNorm_0"]["scale"],
       prefix + "layernorm_after.bias": state["LayerNorm_1"]["bias"],
       prefix + "layernorm_after.weight": state["LayerNorm_1"]["scale"],
       prefix
       + "attention.attention.query.bias": transform_attention(
           state["MultiHeadDotProductAttention_0"]["query"]["bias"]),
       prefix
       + "attention.attention.query.weight": transform_attention(
           state["MultiHeadDotProductAttention_0"]["query"]["kernel"]),
       prefix
       + "attention.attention.key.bias": transform_attention(state["MultiHeadDotProductAttention_0"]["key"]["bias"]),
       prefix
       + "attention.attention.key.weight": transform_attention(
           state["MultiHeadDotProductAttention_0"]["key"]["kernel"]),
       prefix
       + "attention.attention.value.bias": transform_attention(
           state["MultiHeadDotProductAttention_0"]["value"]["bias"]),
       prefix
       + "attention.attention.value.weight": transform_attention(
           state["MultiHeadDotProductAttention_0"]["value"]["kernel"]),
       prefix + "attention.output.dense.bias": state["MultiHeadDotProductAttention_0"]["out"]["bias"],
       prefix
       + "attention.output.dense.weight": transform_attention_output_weight(
           state["MultiHeadDotProductAttention_0"]["out"]["kernel"]),
   }

   return new_state




def 						_A						(						UpperCamelCase_  : str)			->		List[Any]:
   '''simple docstring'''


   return sum([1 if "encoderblock_" in k else 0 for k in state_dict["optimizer"]["target"]["Transformer"].keys()])




def 						_A						(						UpperCamelCase_  : Union[str, Any],    UpperCamelCase_  : Any=False)			->		Any:
   '''simple docstring'''


   __lowercase			=		get_n_layers(UpperCamelCase_)

   __lowercase			=		OrderedDict()

   __lowercase			=		state_dict["optimizer"]["target"]["Transformer"]["encoder_norm"]["bias"]
   __lowercase			=		state_dict["optimizer"]["target"]["Transformer"]["encoder_norm"]["scale"]

   __lowercase			=		np.transpose(
       state_dict["optimizer"]["target"]["embedding"]["kernel"],    (4, 3, 0, 1, 2))
   __lowercase			=		state_dict["optimizer"]["target"]["embedding"]["bias"]

   __lowercase			=		state_dict["optimizer"]["target"]["cls"]
   __lowercase			=		state_dict["optimizer"]["target"]["Transformer"]["posembed_input"][
       "pos_embedding"
   ]

   for i in range(UpperCamelCase_):
      new_state.update(transform_state_encoder_block(UpperCamelCase_,    UpperCamelCase_))

   if classification_head:
      __lowercase			=		{"vivit." + k: v for k, v in new_state.items()}
      __lowercase			=		np.transpose(state_dict["optimizer"]["target"]["output_projection"]["kernel"])
      __lowercase			=		np.transpose(state_dict["optimizer"]["target"]["output_projection"]["bias"])

   return {k: torch.tensor(UpperCamelCase_) for k, v in new_state.items()}




def 						_A						(						)			->		VivitImageProcessor:
   '''simple docstring'''


   __lowercase			=		VivitImageProcessor()

   assert extractor.do_resize is True
   assert extractor.size == {"shortest_edge": 256}
   assert extractor.do_center_crop is True
   assert extractor.crop_size == {"width": 224, "height": 224}
   assert extractor.resample == PILImageResampling.BILINEAR

   # here: https://github.com/deepmind/dmvr/blob/master/dmvr/modalities.py
   # one can seen that add_image has default values for normalization_mean and normalization_std set to 0 and 1
   # which effectively means no normalization (and ViViT does not overwrite those when calling this func)
   assert extractor.do_normalize is False
   assert extractor.do_rescale is True
   assert extractor.rescale_factor == 1 / 255

   # zero-centering = True in original implementation
   assert extractor.do_zero_centering is True

   return extractor




def 						_A						(						UpperCamelCase_  : str)			->		List[str]:
   '''simple docstring'''


   __lowercase			=		"checkpoint"

   if not os.path.exists(UpperCamelCase_):
      download_checkpoint(UpperCamelCase_)

   __lowercase			=		restore_checkpoint(UpperCamelCase_,    UpperCamelCase_)
   __lowercase			=		transform_state(UpperCamelCase_,    classification_head=UpperCamelCase_)

   __lowercase			=		get_vivit_config()

   assert config.image_size == 224
   assert config.num_frames == 32

   __lowercase			=		VivitForVideoClassification(UpperCamelCase_)
   model.load_state_dict(UpperCamelCase_)
   model.eval()

   __lowercase			=		get_processor()

   __lowercase			=		prepare_video()
   __lowercase			=		extractor(UpperCamelCase_,    return_tensors="pt")

   __lowercase			=		model(**UpperCamelCase_)

   __lowercase			=		torch.Size([1, 400])
   __lowercase			=		torch.tensor([-1.0_543, 2.0_764, -0.2_104, 0.4_439, -0.9_658])

   assert outputs.logits.shape == expected_shape
   assert torch.allclose(outputs.logits[0, :5],    UpperCamelCase_,    atol=1E-4), outputs.logits[0, :5]

   model.save_pretrained(UpperCamelCase_)
   extractor.save_pretrained(UpperCamelCase_)


if __name__ == "__main__":
   _a				   =							argparse.ArgumentParser()

   parser.add_argument('--output_model_name', '-o', type=str, help='Output path for the converted HuggingFace model')

   _a				   =							parser.parse_args()
   convert(args.output_model_name)




