




'''simple docstring'''



# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List







def 			_UpperCamelCase (_lowerCamelCase							:     List[str]   ,	_lowerCamelCase							:     str	)->   str:




     '''simple docstring'''





     if name != "training_function" and name != "main":
          raise ValueError(f'''Incorrect function name passed: {name}, choose either \'main\' or \'training_function\''''	)
     __snake_case					,    __snake_case    					=  [], False
     for line in lines:
          if not found_start and f'''def {name}''' in line:
               __snake_case    					=  True
               good_lines.append(_lowerCamelCase	)
               continue
          if found_start:
               if name == "training_function" and "def main" in line:
                    return good_lines
               if name == "main" and "if __name__" in line:
                    return good_lines
               good_lines.append(_lowerCamelCase	)







def 			_UpperCamelCase (_lowerCamelCase							:     List[str]	)->   int:




     '''simple docstring'''





     return [line for line in lines if not line.lstrip().startswith('''#'''	) and line != "\n"]







def 			_UpperCamelCase (_lowerCamelCase							:     str   ,	_lowerCamelCase							:     str   ,	_lowerCamelCase							:     bool   ,	_lowerCamelCase							:     str = None	)->   Optional[Any]:




     '''simple docstring'''





     with open(_lowerCamelCase   ,	'''r'''	) as f:
          __snake_case    					=  f.readlines()
     with open(os.path.abspath(os.path.join('''examples'''   ,	'''nlp_example.py'''	)	)   ,	'''r'''	) as f:
          __snake_case    					=  f.readlines()
     with open(_lowerCamelCase   ,	'''r'''	) as f:
          __snake_case    					=  f.readlines()
     if secondary_filename is not None:
          with open(_lowerCamelCase   ,	'''r'''	) as f:
               __snake_case    					=  f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
     if parser_only:
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
          if secondary_filename is not None:
               __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''main'''	)	)
     else:
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)
          __snake_case    					=  clean_lines(get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)
          if secondary_filename is not None:
               __snake_case    					=  clean_lines(
                   get_function_contents_by_name(_lowerCamelCase   ,	'''training_function'''	)	)

     __snake_case    					=  '''train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n'''

     # Specific code in our script that differs from the full version, aka what is new
     __snake_case    					=  []
     __snake_case    					=  []  # We keep track of the idxs just in case it's a repeated statement
     __snake_case    					=  iter(_lowerCamelCase	)
     for i in range(len(_lowerCamelCase	) - 1	):
          if i not in passed_idxs:
               __snake_case    					=  next(_lowerCamelCase	)
               if (line not in full_file_func) and (line.lstrip() != _dl_line):
                    if "TESTING_MOCKED_DATALOADERS" not in line:
                         new_feature_code.append(_lowerCamelCase	)
                         passed_idxs.append(_lowerCamelCase	)
                    else:
                         # Skip over the `config['num_epochs'] = 2` statement
                         __snake_case    					=  next(_lowerCamelCase	)

    # Extract out just the new parts from the full_file_training_func
     __snake_case    					=  []
     __snake_case    					=  []  # We keep track of the idxs just in case it's a repeated statement
     for i, line in enumerate(_lowerCamelCase	):
          if i not in passed_idxs:
               if (line not in full_file_func) and (line.lstrip() != _dl_line):
                    if "TESTING_MOCKED_DATALOADERS" not in line:
                         new_full_example_parts.append(_lowerCamelCase	)
                         passed_idxs.append(_lowerCamelCase	)

    # Finally, get the overall diff
     __snake_case    					=  [line for line in new_feature_code if line not in new_full_example_parts]
     if secondary_filename is not None:
          __snake_case    					=  [line for line in full_file_contents if line not in secondary_file_func]
          __snake_case    					=  [line for line in diff_from_example if line not in diff_from_two]

     return diff_from_example












'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


UpperCAmelCase_       :  str      =		datasets.utils.logging.get_logger(__name__)





@dataclass
class     lowerCAmelCase   ( datasets.BuilderConfig):
       __lowercase  :      Optional[datasets.Features]        =			None
       __lowercase  :      str        =			"utf-8"
       __lowercase  :      Optional[str]        =			None
       __lowercase  :      int        =			10 << 20  # 10MB
       __lowercase  :      bool        =			False
       __lowercase  :      str        =			"line"





class     lowerCAmelCase   ( datasets.ArrowBasedBuilder):
       __lowercase  :      Dict        =			TextConfig


       def       lowerCAmelCase      (   self  )      ->  Dict:



            '''simple docstring'''




            return datasets.DatasetInfo(features=self.config.features  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
            __snake_case    					=  dl_manager.download_and_extract(self.config.data_files  )
            if isinstance(__SCREAMING_SNAKE_CASE    ,  (str, list, tuple)  ):
                 __snake_case    					=  data_files
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,  gen_kwargs={'''files''': files}  )]
            __snake_case    					=  []
            for split_name, files in data_files.items():
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE    ,  gen_kwargs={'''files''': files}  )  )
            return splits


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  pa.Table:



            '''simple docstring'''




            if self.config.features is not None:
                 __snake_case    					=  self.config.features.arrow_schema
                 if all(not require_storage_cast(__SCREAMING_SNAKE_CASE  ) for feature in self.config.features.values()  ):
                      # cheaper cast
                      __snake_case    					=  pa_table.cast(__SCREAMING_SNAKE_CASE  )
                 else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      __snake_case    					=  table_cast(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
                 return pa_table
            else:
                 return pa_table.cast(pa.schema({'''text''': pa.string()}  )  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  list(self.config.features  ) if self.config.features is not None else ['''text''']
            for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE  )  ):
                 # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
                 with open(__SCREAMING_SNAKE_CASE    ,  encoding=self.config.encoding    ,  errors=self.config.errors  ) as f:
                      if self.config.sample_by == "line":
                           __snake_case    					=  0
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not batch:
                                     break
                                batch += f.readline()  # finish current line
                                # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                __snake_case    					=  StringIO(__SCREAMING_SNAKE_CASE  ).readlines()
                                if not self.config.keep_linebreaks:
                                     __snake_case    					=  [line.rstrip('''\n'''  ) for line in batch]
                                __snake_case    					=  pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                           __snake_case    					=  0
                           __snake_case    					=  ''''''
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not new_batch:
                                     break
                                batch += new_batch
                                batch += f.readline()  # finish current line
                                __snake_case    					=  batch.split('''\n\n'''  )
                                __snake_case    					=  pa.Table.from_arrays(
                                    [pa.array([example for example in batch[:-1] if example]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                                __snake_case    					=  batch[-1]
                           if batch:
                                __snake_case    					=  pa.Table.from_arrays([pa.array([batch]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                      elif self.config.sample_by == "document":
                           __snake_case    					=  f.read()
                           __snake_case    					=  pa.Table.from_arrays([pa.array([text]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                           yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE  )






