





import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch





def   _lowercase       (    )    ->	Optional[int]:
				lowerCamelCase									=Dataset.from_dict({"""i""": range(17					)}					)
				lowerCamelCase									=len(_UpperCAmelCase					)
				lowerCamelCase									=3
				lowerCamelCase									=[
				    split_dataset_by_node(_UpperCAmelCase      ,   rank=_UpperCAmelCase      ,   world_size=_UpperCAmelCase					) for rank in range(_UpperCAmelCase					)
				]
				assert sum(len(_UpperCAmelCase					) for ds in datasets_per_rank					) == full_size
				assert len({tuple(x.values()					) for ds in datasets_per_rank for x in ds}					) == full_size





def   _lowercase       (    )    ->	List[Any]:
				def gen():
								return ({"i": i} for i in range(17					))

				lowerCamelCase									=3
				lowerCamelCase									=IterableDataset.from_generator(_UpperCAmelCase					)
				lowerCamelCase									=len(list(_UpperCAmelCase					)					)
				lowerCamelCase									=[
				    split_dataset_by_node(_UpperCAmelCase      ,   rank=_UpperCAmelCase      ,   world_size=_UpperCAmelCase					) for rank in range(_UpperCAmelCase					)
				]
				assert sum(len(list(_UpperCAmelCase					)					) for ds in datasets_per_rank					) == full_size
				assert len({tuple(x.values()					) for ds in datasets_per_rank for x in ds}					) == full_size





@pytest.mark.parametrize("""shards_per_node"""      ,   [1, 2, 3]					)
def   _lowercase       (    _UpperCAmelCase					)    ->	List[Any]:
				def gen(_UpperCAmelCase					):
								for shard in shards:
												yield from ({"i": i, "shard": shard} for i in range(17					))

				lowerCamelCase									=3
				lowerCamelCase									=shards_per_node * world_size
				lowerCamelCase									={"""shards""": [F"""shard_{shard_idx}.txt""" for shard_idx in range(_UpperCAmelCase					)]}
				lowerCamelCase									=IterableDataset.from_generator(_UpperCAmelCase      ,   gen_kwargs=_UpperCAmelCase					)
				lowerCamelCase									=len(list(_UpperCAmelCase					)					)
				assert full_ds.n_shards == world_size * shards_per_node
				lowerCamelCase									=[
				    split_dataset_by_node(_UpperCAmelCase      ,   rank=_UpperCAmelCase      ,   world_size=_UpperCAmelCase					) for rank in range(_UpperCAmelCase					)
				]
				assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
				assert sum(len(list(_UpperCAmelCase					)					) for ds in datasets_per_rank					) == full_size
				assert len({tuple(x.values()					) for ds in datasets_per_rank for x in ds}					) == full_size





def   _lowercase       (    )    ->	List[str]:
				def gen():
								return ({"i": i} for i in range(17					))

				lowerCamelCase									=2
				lowerCamelCase									=IterableDataset.from_generator(_UpperCAmelCase					)
				lowerCamelCase									=len(list(_UpperCAmelCase					)					)

				lowerCamelCase									=split_dataset_by_node(_UpperCAmelCase      ,   rank=0      ,   world_size=_UpperCAmelCase					).shuffle(seed=42					)
				assert len(list(_UpperCAmelCase					)					) == 1 + full_size // world_size
				with pytest.raises(_UpperCAmelCase					):
								split_dataset_by_node(_UpperCAmelCase      ,   rank=0      ,   world_size=_UpperCAmelCase					).shuffle()

				lowerCamelCase									=split_dataset_by_node(full_ds.shuffle(seed=42					)      ,   rank=0      ,   world_size=_UpperCAmelCase					)
				assert len(list(_UpperCAmelCase					)					) == 1 + full_size // world_size
				with pytest.raises(_UpperCAmelCase					):
								split_dataset_by_node(full_ds.shuffle()      ,   rank=0      ,   world_size=_UpperCAmelCase					)





@pytest.mark.parametrize("""streaming"""      ,   [False, True]					)
@require_torch
@pytest.mark.skipif(os.name == """nt"""      ,   reason="""execute_subprocess_async doesn't support windows"""					)
@pytest.mark.integration
def   _lowercase       (    _UpperCAmelCase					)    ->	Optional[int]:
				lowerCamelCase									=2
				lowerCamelCase									=get_torch_dist_unique_port()
				lowerCamelCase									=Path(_UpperCAmelCase					).resolve().parent / """distributed_scripts""" / """launch_torch_distributed.py"""
				lowerCamelCase									=F"""
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    """.split()
				lowerCamelCase									=F"""
        --streaming={streaming}
    """.split()
				lowerCamelCase									=[sys.executable] + distributed_args + args
				execute_subprocess_async(_UpperCAmelCase      ,   env=os.environ.copy()					)





@pytest.mark.parametrize(
    """nproc_per_node, num_workers"""      ,   [
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ]      ,   )
@require_torch
@pytest.mark.skipif(os.name == """nt"""      ,   reason="""execute_subprocess_async doesn't support windows"""					)
@pytest.mark.integration
def   _lowercase       (    _UpperCAmelCase      ,   _UpperCAmelCase					)    ->	str:
				lowerCamelCase									=True
				lowerCamelCase									=get_torch_dist_unique_port()
				lowerCamelCase									=Path(_UpperCAmelCase					).resolve().parent / """distributed_scripts""" / """launch_torch_distributed.py"""
				lowerCamelCase									=F"""
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    """.split()
				lowerCamelCase									=F"""
        --streaming={streaming}
        --num_workers={num_workers}
    """.split()
				lowerCamelCase									=[sys.executable] + distributed_args + args
				execute_subprocess_async(_UpperCAmelCase      ,   env=os.environ.copy()					)












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


UpperCAmelCase__    :      int       					=logging.get_logger(__name__)

UpperCAmelCase__    :      Optional[Any]       					={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__    :      List[str]       					={
    '''vocab_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__    :      Dict       					={
    '''microsoft/layoutlm-base-uncased''': 5_12,
    '''microsoft/layoutlm-large-uncased''': 5_12,
}

UpperCAmelCase__    :      List[Any]       					={
    '''microsoft/layoutlm-base-uncased''': {'''do_lower_case''': True},
    '''microsoft/layoutlm-large-uncased''': {'''do_lower_case''': True},
}

class    __A  (    a     ):
				__A  =					VOCAB_FILES_NAMES
				__A  =					PRETRAINED_VOCAB_FILES_MAP
				__A  =					PRETRAINED_INIT_CONFIGURATION
				__A  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A  =					LayoutLMTokenizer


				def __init__(		self ,			UpperCAmelCase_=None ,			UpperCAmelCase_=None ,			UpperCAmelCase_=True ,			UpperCAmelCase_="[UNK]" ,			UpperCAmelCase_="[SEP]" ,			UpperCAmelCase_="[PAD]" ,			UpperCAmelCase_="[CLS]" ,			UpperCAmelCase_="[MASK]" ,			UpperCAmelCase_=True ,			UpperCAmelCase_=None ,			**UpperCAmelCase_ ,			):
								super().__init__(
								    UpperCAmelCase_ ,			tokenizer_file=UpperCAmelCase_ ,			do_lower_case=UpperCAmelCase_ ,			unk_token=UpperCAmelCase_ ,			sep_token=UpperCAmelCase_ ,			pad_token=UpperCAmelCase_ ,			cls_token=UpperCAmelCase_ ,			mask_token=UpperCAmelCase_ ,			tokenize_chinese_chars=UpperCAmelCase_ ,			strip_accents=UpperCAmelCase_ ,			**UpperCAmelCase_ ,			)

								lowerCamelCase									=json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    normalizer_state.get("""lowercase""" ,			UpperCAmelCase_				) != do_lower_case
								    or normalizer_state.get("""strip_accents""" ,			UpperCAmelCase_				) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars""" ,			UpperCAmelCase_				) != tokenize_chinese_chars
								):
												lowerCamelCase									=getattr(UpperCAmelCase_ ,			normalizer_state.pop("""type"""				)				)
												lowerCamelCase									=do_lower_case
												lowerCamelCase									=strip_accents
												lowerCamelCase									=tokenize_chinese_chars
												lowerCamelCase									=normalizer_class(**UpperCAmelCase_				)

								lowerCamelCase									=do_lower_case


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_=None				):
								lowerCamelCase									=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = None				):
								lowerCamelCase									=[self.sep_token_id]
								lowerCamelCase									=[self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep				) * [0]
								return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]


				def _snake_case							(		self ,			UpperCAmelCase_ ,			UpperCAmelCase_ = None				):
								lowerCamelCase									=self._tokenizer.model.save(UpperCAmelCase_ ,			name=UpperCAmelCase_				)
								return tuple(UpperCAmelCase_				)





