


"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


SCREAMING_SNAKE_CASE__				:  List[Any]						= {
    """configuration_markuplm""": ["""MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MarkupLMConfig"""],
    """feature_extraction_markuplm""": ["""MarkupLMFeatureExtractor"""],
    """processing_markuplm""": ["""MarkupLMProcessor"""],
    """tokenization_markuplm""": ["""MarkupLMTokenizer"""],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE__				:  Optional[Any]						= ["""MarkupLMTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE__				:  Optional[Any]						= [
     """MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """MarkupLMForQuestionAnswering""",
     """MarkupLMForSequenceClassification""",
     """MarkupLMForTokenClassification""",
     """MarkupLMModel""",
     """MarkupLMPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_markuplm import MARKUPLM_PRETRAINED_CONFIG_ARCHIVE_MAP, MarkupLMConfig
 from .feature_extraction_markuplm import MarkupLMFeatureExtractor
 from .processing_markuplm import MarkupLMProcessor
 from .tokenization_markuplm import MarkupLMTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_markuplm_fast import MarkupLMTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_markuplm import (
      MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LIST,
      MarkupLMForQuestionAnswering,
      MarkupLMForSequenceClassification,
      MarkupLMForTokenClassification,
      MarkupLMModel,
      MarkupLMPreTrainedModel,
  )


else:
 import sys

 SCREAMING_SNAKE_CASE__				:  int						= _LazyModule(__name__, globals()["__file__"], _import_structure)













from transformers import (
    DPRContextEncoderTokenizer,
    DPRContextEncoderTokenizerFast,
    DPRQuestionEncoderTokenizer,
    DPRQuestionEncoderTokenizerFast,
    DPRReaderOutput,
    DPRReaderTokenizer,
    DPRReaderTokenizerFast,
)
from transformers.testing_utils import require_tokenizers, slow
from transformers.tokenization_utils_base import BatchEncoding

from ..bert.test_tokenization_bert import BertTokenizationTest


@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Tuple				=       DPRContextEncoderTokenizer
						_SCREAMING_SNAKE_CASE						:							Any				=       DPRContextEncoderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							str				=       True




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Optional[Any]				=       DPRQuestionEncoderTokenizer
						_SCREAMING_SNAKE_CASE						:							List[Any]				=       DPRQuestionEncoderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							Optional[Any]				=       True




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Any				=       DPRReaderTokenizer
						_SCREAMING_SNAKE_CASE						:							int				=       DPRReaderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							List[str]				=       True






						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tokenizer_class.from_pretrained('bert-base-uncased' )

												lowerCAmelCase__         =    tokenizer.encode('question sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('title sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('text sequence ' * 4  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    [[1_01] + text_a + [1_02] + text_a + [1_02] + text_a]
												lowerCAmelCase__         =    BatchEncoding({'input_ids': input_ids} )

												lowerCAmelCase__         =    [[0] * len(input_ids[0] )]
												lowerCAmelCase__         =    [[0] * len(input_ids[0] )]
												lowerCAmelCase__         =    [0]
												lowerCAmelCase__         =    DPRReaderOutput(_UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase )

												lowerCAmelCase__    ,							lowerCAmelCase__         =    8, 9
												lowerCAmelCase__         =    10
												lowerCAmelCase__         =    10
												lowerCAmelCase__         =    tokenizer.decode_best_spans(_UpperCamelCase  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].start_index  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].end_index  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].doc_id  ,    0 )







						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tokenizer_class.from_pretrained('bert-base-uncased' )

												lowerCAmelCase__         =    tokenizer.encode('question sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('title sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('text sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    [1_01] + text_a + [1_02] + text_a + [1_02] + text_a
												lowerCAmelCase__         =    tokenizer(questions=['question sequence']  ,    titles=['title sequence']  ,    texts=['text sequence'] )
												self.assertIn('input_ids'  ,    _UpperCamelCase )
												self.assertIn('attention_mask'  ,    _UpperCamelCase )
												self.assertListEqual(encoded_input['input_ids'][0]  ,    _UpperCamelCase )





