



"""simple docstring"""
import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json





@pytest.fixture
def 				lowerCamelCase_		(UpperCamelCase__	:	Optional[Any]							):
    _UpperCAmelCase							:						List[str]       							=					tmp_path / '''file.jsonl'''
    _UpperCAmelCase							:						Union[str, Any]       							=					textwrap.dedent(
        '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''							)
    with open(UpperCamelCase__  , '''w'''							) as f:
        f.write(UpperCamelCase__							)
    return str(UpperCamelCase__							)





@pytest.fixture
def 				lowerCamelCase_		(UpperCamelCase__	:	List[Any]							):
    _UpperCAmelCase							:						int       							=					tmp_path / '''file_utf16_encoded.jsonl'''
    _UpperCAmelCase							:						Tuple       							=					textwrap.dedent(
        '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''							)
    with open(UpperCamelCase__  , '''w'''  , encoding='''utf-16'''							) as f:
        f.write(UpperCamelCase__							)
    return str(UpperCamelCase__							)





@pytest.fixture
def 				lowerCamelCase_		(UpperCamelCase__	:	Union[str, Any]							):
    _UpperCAmelCase							:						Tuple       							=					tmp_path / '''file_with_list_of_dicts.json'''
    _UpperCAmelCase							:						Optional[Any]       							=					textwrap.dedent(
        '''\
        [
            {"col_1": -1},
            {"col_1": 1, "col_2": 2},
            {"col_1": 10, "col_2": 20}
        ]
        '''							)
    with open(UpperCamelCase__  , '''w'''							) as f:
        f.write(UpperCamelCase__							)
    return str(UpperCamelCase__							)





@pytest.fixture
def 				lowerCamelCase_		(UpperCamelCase__	:	Tuple							):
    _UpperCAmelCase							:						Optional[int]       							=					tmp_path / '''file_with_list_of_dicts_field.json'''
    _UpperCAmelCase							:						Union[str, Any]       							=					textwrap.dedent(
        '''\
        {
            "field1": 1,
            "field2": "aabb",
            "field3": [
                {"col_1": -1},
                {"col_1": 1, "col_2": 2},
                {"col_1": 10, "col_2": 20}
            ]
        }
        '''							)
    with open(UpperCamelCase__  , '''w'''							) as f:
        f.write(UpperCamelCase__							)
    return str(UpperCamelCase__							)





@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''  , [
        ('''jsonl_file''', {}),
        ('''jsonl_file_utf16_encoded''', {'''encoding''': '''utf-16'''}),
        ('''json_file_with_list_of_dicts''', {}),
        ('''json_file_with_list_of_dicts_field''', {'''field''': '''field3'''}),
    ]  , )
def 				lowerCamelCase_		(UpperCamelCase__	:	Optional[int]  , UpperCamelCase__	:	Union[str, Any]  , UpperCamelCase__	:	Tuple							):
    _UpperCAmelCase							:						Any       							=					Json(**UpperCamelCase__							)
    _UpperCAmelCase							:						str       							=					json._generate_tables([[request.getfixturevalue(UpperCamelCase__							)]]							)
    _UpperCAmelCase							:						Optional[Any]       							=					pa.concat_tables([table for _, table in generator]							)
    assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}





@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''  , [
        (
            '''jsonl_file''',
            {'''features''': Features({'''col_1''': Value('''int64'''							), '''col_2''': Value('''int64'''							), '''missing_col''': Value('''string'''							)}							)},
        ),
        (
            '''json_file_with_list_of_dicts''',
            {'''features''': Features({'''col_1''': Value('''int64'''							), '''col_2''': Value('''int64'''							), '''missing_col''': Value('''string'''							)}							)},
        ),
        (
            '''json_file_with_list_of_dicts_field''',
            {
                '''field''': '''field3''',
                '''features''': Features(
                    {'''col_1''': Value('''int64'''							), '''col_2''': Value('''int64'''							), '''missing_col''': Value('''string'''							)}							),
            },
        ),
    ]  , )
def 				lowerCamelCase_		(UpperCamelCase__	:	Dict  , UpperCamelCase__	:	List[str]  , UpperCamelCase__	:	Optional[Any]							):
    _UpperCAmelCase							:						List[Any]       							=					Json(**UpperCamelCase__							)
    _UpperCAmelCase							:						Any       							=					json._generate_tables([[request.getfixturevalue(UpperCamelCase__							)]]							)
    _UpperCAmelCase							:						List[Any]       							=					pa.concat_tables([table for _, table in generator]							)
    assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}







"""simple docstring"""
import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer
class 							_UpperCAmelCase							(				tf.keras.layers.Layer   ):





     '''simple docstring'''



     def __init__(      self ,			A ,			A ,			A = None ,			A = None ,			A = None ,			A = "longest" ,			A = True ,			A = 5_1_2 ,			A = None ,			A = True ,			A = True ,			A = True ,			**A ,			)							->      Tuple:
         super().__init__()
         if use_fast_bert_tokenizer:
             _UpperCAmelCase							:						Union[str, Any]       							=					FastBertTokenizer(
                 A ,			token_out_type=tf.intaa ,			lower_case_nfd_strip_accents=A ,			**A	)
         else:
             _UpperCAmelCase							:						List[Any]       							=					tf.lookup.StaticVocabularyTable(
                 tf.lookup.KeyValueTensorInitializer(
                     keys=A ,			key_dtype=tf.string ,			values=tf.range(tf.size(A ,			out_type=tf.intaa	) ,			dtype=tf.intaa	) ,			value_dtype=tf.intaa ,			) ,			num_oov_buckets=1 ,			)
             _UpperCAmelCase							:						str       							=					BertTokenizerLayer(
                 A ,			token_out_type=tf.intaa ,			lower_case=A ,			**A	)

         _UpperCAmelCase							:						int       							=					vocab_list
         _UpperCAmelCase							:						Dict       							=					do_lower_case
         _UpperCAmelCase							:						Optional[int]       							=					cls_token_id or vocab_list.index('''[CLS]'''	)
         _UpperCAmelCase							:						Any       							=					sep_token_id or vocab_list.index('''[SEP]'''	)
         _UpperCAmelCase							:						List[Any]       							=					pad_token_id or vocab_list.index('''[PAD]'''	)
         _UpperCAmelCase							:						List[str]       							=					ShrinkLongestTrimmer(max_length - 3 ,			axis=1	)  # Allow room for special tokens
         _UpperCAmelCase							:						Tuple       							=					max_length
         _UpperCAmelCase							:						List[Any]       							=					padding
         _UpperCAmelCase							:						List[Any]       							=					truncation
         _UpperCAmelCase							:						Union[str, Any]       							=					pad_to_multiple_of
         _UpperCAmelCase							:						Union[str, Any]       							=					return_token_type_ids
         _UpperCAmelCase							:						Union[str, Any]       							=					return_attention_mask





     @classmethod
     def 				__lowerCAmelCase			(      cls ,			A ,			**A	)							->      Dict:  # noqa: F821
         _UpperCAmelCase							:						Optional[int]       							=					kwargs.pop('''do_lower_case''' ,			A	)
         _UpperCAmelCase							:						int       							=					tokenizer.do_lower_case if do_lower_case is None else do_lower_case
         _UpperCAmelCase							:						Optional[Any]       							=					kwargs.pop('''cls_token_id''' ,			A	)
         _UpperCAmelCase							:						str       							=					tokenizer.cls_token_id if cls_token_id is None else cls_token_id
         _UpperCAmelCase							:						str       							=					kwargs.pop('''sep_token_id''' ,			A	)
         _UpperCAmelCase							:						int       							=					tokenizer.sep_token_id if sep_token_id is None else sep_token_id
         _UpperCAmelCase							:						List[Any]       							=					kwargs.pop('''pad_token_id''' ,			A	)
         _UpperCAmelCase							:						Optional[Any]       							=					tokenizer.pad_token_id if pad_token_id is None else pad_token_id

         _UpperCAmelCase							:						int       							=					tokenizer.get_vocab()
         _UpperCAmelCase							:						List[str]       							=					sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()] ,			key=lambda A	: x[1]	)
         _UpperCAmelCase							:						Dict       							=					[entry[0] for entry in vocab]
         return cls(
             vocab_list=A ,			do_lower_case=A ,			cls_token_id=A ,			sep_token_id=A ,			pad_token_id=A ,			**A ,			)





     @classmethod
     def 				__lowerCAmelCase			(      cls ,			A ,			*A ,			**A	)							->      int:
         try:
             _UpperCAmelCase							:						int       							=					BertTokenizer.from_pretrained(A ,			*A ,			**A	)
         except:  # noqa: E722
             from .tokenization_bert_fast import BertTokenizerFast

             _UpperCAmelCase							:						Dict       							=					BertTokenizerFast.from_pretrained(A ,			*A ,			**A	)
         return cls.from_tokenizer(A ,			**A	)





     def 				__lowerCAmelCase			(      self ,			A	)							->      Any:
         if self.do_lower_case:
             _UpperCAmelCase							:						int       							=					case_fold_utfa(A	)
         _UpperCAmelCase							:						str       							=					self.tf_tokenizer.tokenize(A	)
         return tokens.merge_dims(1 ,			-1	)





     def 				__lowerCAmelCase			(      self ,			A ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			)							->      Optional[Any]:
         if padding is None:
             _UpperCAmelCase							:						str       							=					self.padding
         if padding not in ("longest", "max_length"):
             raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''	)
         if max_length is not None and text_pair is not None:
             # Because we have to instantiate a Trimmer to do it properly
             raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''	)
         if max_length is None:
             _UpperCAmelCase							:						Union[str, Any]       							=					self.max_length
         if truncation is None:
             _UpperCAmelCase							:						List[Any]       							=					self.truncation
         if pad_to_multiple_of is None:
             _UpperCAmelCase							:						List[Any]       							=					self.pad_to_multiple_of
         if return_token_type_ids is None:
             _UpperCAmelCase							:						List[str]       							=					self.return_token_type_ids
         if return_attention_mask is None:
             _UpperCAmelCase							:						Any       							=					self.return_attention_mask
         if not isinstance(A ,			tf.Tensor	):
             _UpperCAmelCase							:						Dict       							=					tf.convert_to_tensor(A	)
         if text_pair is not None and not isinstance(A ,			tf.Tensor	):
             _UpperCAmelCase							:						str       							=					tf.convert_to_tensor(A	)
         if text_pair is not None:
             if text.shape.rank > 1:
                 raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''	)
             if text_pair.shape.rank > 1:
                 raise ValueError('''text_pair should not be multidimensional!'''	)
         if text.shape.rank == 2:
             _UpperCAmelCase							,       _UpperCAmelCase							:						str       							=					text[:, 0], text[:, 1]
         _UpperCAmelCase							:						Optional[Any]       							=					self.unpaired_tokenize(A	)
         if text_pair is None:  # Unpaired text
             if truncation:
                 _UpperCAmelCase							:						Optional[int]       							=					text[:, : max_length - 2]  # Allow room for special tokens
             _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					combine_segments(
                 (text,) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
         else:  # Paired text
             _UpperCAmelCase							:						Tuple       							=					self.unpaired_tokenize(A	)
             if truncation:
                 _UpperCAmelCase							,       _UpperCAmelCase							:						Optional[int]       							=					self.paired_trimmer.trim([text, text_pair]	)
             _UpperCAmelCase							,       _UpperCAmelCase							:						Union[str, Any]       							=					combine_segments(
                 (text, text_pair) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
         if padding == "longest":
             _UpperCAmelCase							:						List[Any]       							=					input_ids.bounding_shape(axis=1	)
             if pad_to_multiple_of is not None:
                 # No ceiling division in tensorflow, so we negate floordiv instead
                 _UpperCAmelCase							:						Tuple       							=					pad_to_multiple_of * (-tf.math.floordiv(-pad_length ,			A	))
         else:
             _UpperCAmelCase							:						int       							=					max_length

         _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					pad_model_inputs(A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
         _UpperCAmelCase							:						Dict       							=					{'''input_ids''': input_ids}
         if return_attention_mask:
             _UpperCAmelCase							:						Optional[int]       							=					attention_mask
         if return_token_type_ids:
             _UpperCAmelCase							,       _UpperCAmelCase							:						Tuple       							=					pad_model_inputs(
                 A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
             _UpperCAmelCase							:						int       							=					token_type_ids
         return output





     def 				__lowerCAmelCase			(      self	)							->      List[str]:
         return {
             "vocab_list": self.vocab_list,
             "do_lower_case": self.do_lower_case,
             "cls_token_id": self.cls_token_id,
             "sep_token_id": self.sep_token_id,
             "pad_token_id": self.pad_token_id,
         }


