Windows Permission Error (most recent version of datasets)
 Hi everyone,
Can anyone help me with why the dataset loading script below raises a Windows Permission Error? I stuck quite closely to https://github.com/huggingface/datasets/blob/master/datasets/conll2003/conll2003.py , only I want to load the data from three local three-column tsv-files (id\ttokens\tpos_tags\n). I am using the most recent version of datasets. Thank you in advance!
Luisa

My script:
```
import datasets
import csv

logger = datasets.logging.get_logger(__name__)

class SampleConfig(datasets.BuilderConfig):

    def __init__(self, **kwargs):
        super(SampleConfig, self).__init__(**kwargs)

class Sample(datasets.GeneratorBasedBuilder):
    BUILDER_CONFIGS = [
        SampleConfig(name="conll2003", version=datasets.Version("1.0.0"), description="Conll2003 dataset"),
    ]

    def _info(self):
        return datasets.DatasetInfo(
            description="Dataset with words and their POS-Tags",
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "pos_tags": datasets.Sequence(
                        datasets.features.ClassLabel(
                            names=[
                                "''",
                                ",",
                                "-LRB-",
                                "-RRB-",
                                ".",
                                ":",
                                "CC",
                                "CD",
                                "DT",
                                "EX",
                                "FW",
                                "HYPH",
                                "IN",
                                "JJ",
                                "JJR",
                                "JJS",
                                "MD",
                                "NN",
                                "NNP",
                                "NNPS",
                                "NNS",
                                "PDT",
                                "POS",
                                "PRP",
                                "PRP$",
                                "RB",
                                "RBR",
                                "RBS",
                                "RP",
                                "TO",
                                "UH",
                                "VB",
                                "VBD",
                                "VBG",
                                "VBN",
                                "VBP",
                                "VBZ",
                                "WDT",
                                "WP",
                                "WRB",
                                "``"
                            ]
                        )
                    ),
                }
            ),
            supervised_keys=None,
            homepage="https://catalog.ldc.upenn.edu/LDC2011T03",
            citation="Weischedel, Ralph, et al. OntoNotes Release 4.0 LDC2011T03. Web Download. Philadelphia: Linguistic Data Consortium, 2011.",
        )

    def _split_generators(self, dl_manager):
        loaded_files = dl_manager.download_and_extract(self.config.data_files)
        return [
            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": loaded_files["train"]}),
            datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepath": loaded_files["test"]}),
            datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={"filepath": loaded_files["val"]})
        ]

    def _generate_examples(self, filepath):
        logger.info("generating examples from = %s", filepath)
        with open(filepath, encoding="cp1252") as f:
            data = csv.reader(f, delimiter="\t")
            ids = list()
            tokens = list()
            pos_tags = list()
            for id_, line in enumerate(data):
                #print(line)
                if len(line) == 1:
                    if tokens:
                        yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}
                        ids = list()
                        tokens = list()
                        pos_tags = list()
                else:
                    ids.append(line[0])
                    tokens.append(line[1])
                    pos_tags.append(line[2])
            # last example
            yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}

def main():
    dataset = datasets.load_dataset(
        "data_loading.py", data_files={
            "train": "train.tsv",
            "test": "test.tsv",
            "val": "val.tsv"
        }
    )

    #print(dataset)

if __name__=="__main__":
    main()
```

 Hi @itsLuisa !

Could you give us more information about the error you're getting, please?
A copy-paste of the Traceback would be nice to get a better understanding of what is wrong :)


Windows Permission Error (most recent version of datasets)
 Hi everyone,
Can anyone help me with why the dataset loading script below raises a Windows Permission Error? I stuck quite closely to https://github.com/huggingface/datasets/blob/master/datasets/conll2003/conll2003.py , only I want to load the data from three local three-column tsv-files (id\ttokens\tpos_tags\n). I am using the most recent version of datasets. Thank you in advance!
Luisa

My script:
```
import datasets
import csv

logger = datasets.logging.get_logger(__name__)

class SampleConfig(datasets.BuilderConfig):

    def __init__(self, **kwargs):
        super(SampleConfig, self).__init__(**kwargs)

class Sample(datasets.GeneratorBasedBuilder):
    BUILDER_CONFIGS = [
        SampleConfig(name="conll2003", version=datasets.Version("1.0.0"), description="Conll2003 dataset"),
    ]

    def _info(self):
        return datasets.DatasetInfo(
            description="Dataset with words and their POS-Tags",
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "pos_tags": datasets.Sequence(
                        datasets.features.ClassLabel(
                            names=[
                                "''",
                                ",",
                                "-LRB-",
                                "-RRB-",
                                ".",
                                ":",
                                "CC",
                                "CD",
                                "DT",
                                "EX",
                                "FW",
                                "HYPH",
                                "IN",
                                "JJ",
                                "JJR",
                                "JJS",
                                "MD",
                                "NN",
                                "NNP",
                                "NNPS",
                                "NNS",
                                "PDT",
                                "POS",
                                "PRP",
                                "PRP$",
                                "RB",
                                "RBR",
                                "RBS",
                                "RP",
                                "TO",
                                "UH",
                                "VB",
                                "VBD",
                                "VBG",
                                "VBN",
                                "VBP",
                                "VBZ",
                                "WDT",
                                "WP",
                                "WRB",
                                "``"
                            ]
                        )
                    ),
                }
            ),
            supervised_keys=None,
            homepage="https://catalog.ldc.upenn.edu/LDC2011T03",
            citation="Weischedel, Ralph, et al. OntoNotes Release 4.0 LDC2011T03. Web Download. Philadelphia: Linguistic Data Consortium, 2011.",
        )

    def _split_generators(self, dl_manager):
        loaded_files = dl_manager.download_and_extract(self.config.data_files)
        return [
            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": loaded_files["train"]}),
            datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepath": loaded_files["test"]}),
            datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={"filepath": loaded_files["val"]})
        ]

    def _generate_examples(self, filepath):
        logger.info("generating examples from = %s", filepath)
        with open(filepath, encoding="cp1252") as f:
            data = csv.reader(f, delimiter="\t")
            ids = list()
            tokens = list()
            pos_tags = list()
            for id_, line in enumerate(data):
                #print(line)
                if len(line) == 1:
                    if tokens:
                        yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}
                        ids = list()
                        tokens = list()
                        pos_tags = list()
                else:
                    ids.append(line[0])
                    tokens.append(line[1])
                    pos_tags.append(line[2])
            # last example
            yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}

def main():
    dataset = datasets.load_dataset(
        "data_loading.py", data_files={
            "train": "train.tsv",
            "test": "test.tsv",
            "val": "val.tsv"
        }
    )

    #print(dataset)

if __name__=="__main__":
    main()
```

 Hello @SBrandeis , this is it:
```
Traceback (most recent call last):
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 537, in incomplete_dir
    yield tmp_dir
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 578, in download_and_prepare
    self._download_and_prepare(
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 656, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 982, in _prepare_split
    num_examples, num_bytes = writer.finalize()
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\arrow_writer.py", line 297, in finalize
    self.write_on_file()
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\arrow_writer.py", line 230, in write_on_file
    pa_array = pa.array(typed_sequence)
  File "pyarrow\array.pxi", line 222, in pyarrow.lib.array
  File "pyarrow\array.pxi", line 110, in pyarrow.lib._handle_arrow_array_protocol
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\arrow_writer.py", line 97, in __arrow_array__
    out = pa.array(self.data, type=type)
  File "pyarrow\array.pxi", line 305, in pyarrow.lib.array
  File "pyarrow\array.pxi", line 39, in pyarrow.lib._sequence_to_array
  File "pyarrow\error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow\error.pxi", line 107, in pyarrow.lib.check_status
pyarrow.lib.ArrowTypeError: Expected bytes, got a 'list' object

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:/Users/Luisa/Documents/Uni/WS 2020,21/Neural Networks/Final_Project/NN_Project/data_loading.py", line 122, in <module>
    main()
  File "C:/Users/Luisa/Documents/Uni/WS 2020,21/Neural Networks/Final_Project/NN_Project/data_loading.py", line 111, in main
    dataset = datasets.load_dataset(
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 586, in download_and_prepare
    self._save_info()
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 543, in incomplete_dir
    shutil.rmtree(tmp_dir)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\shutil.py", line 740, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\shutil.py", line 618, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\shutil.py", line 616, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 32] Der Prozess kann nicht auf die Datei zugreifen, da sie von einem anderen Prozess verwendet wird: 'C:\\Users\\Luisa\\.cache\\huggingface\\datasets\\sample\\default-20ee7d51a6a9454f\\0.0.0\\5fc4c3a355ea77ab446bd31fca5082437600b8364d29b2b95264048bd1f398b1.incomplete\\sample-train.arrow'

Process finished with exit code 1
```


Windows Permission Error (most recent version of datasets)
 Hi everyone,
Can anyone help me with why the dataset loading script below raises a Windows Permission Error? I stuck quite closely to https://github.com/huggingface/datasets/blob/master/datasets/conll2003/conll2003.py , only I want to load the data from three local three-column tsv-files (id\ttokens\tpos_tags\n). I am using the most recent version of datasets. Thank you in advance!
Luisa

My script:
```
import datasets
import csv

logger = datasets.logging.get_logger(__name__)

class SampleConfig(datasets.BuilderConfig):

    def __init__(self, **kwargs):
        super(SampleConfig, self).__init__(**kwargs)

class Sample(datasets.GeneratorBasedBuilder):
    BUILDER_CONFIGS = [
        SampleConfig(name="conll2003", version=datasets.Version("1.0.0"), description="Conll2003 dataset"),
    ]

    def _info(self):
        return datasets.DatasetInfo(
            description="Dataset with words and their POS-Tags",
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "pos_tags": datasets.Sequence(
                        datasets.features.ClassLabel(
                            names=[
                                "''",
                                ",",
                                "-LRB-",
                                "-RRB-",
                                ".",
                                ":",
                                "CC",
                                "CD",
                                "DT",
                                "EX",
                                "FW",
                                "HYPH",
                                "IN",
                                "JJ",
                                "JJR",
                                "JJS",
                                "MD",
                                "NN",
                                "NNP",
                                "NNPS",
                                "NNS",
                                "PDT",
                                "POS",
                                "PRP",
                                "PRP$",
                                "RB",
                                "RBR",
                                "RBS",
                                "RP",
                                "TO",
                                "UH",
                                "VB",
                                "VBD",
                                "VBG",
                                "VBN",
                                "VBP",
                                "VBZ",
                                "WDT",
                                "WP",
                                "WRB",
                                "``"
                            ]
                        )
                    ),
                }
            ),
            supervised_keys=None,
            homepage="https://catalog.ldc.upenn.edu/LDC2011T03",
            citation="Weischedel, Ralph, et al. OntoNotes Release 4.0 LDC2011T03. Web Download. Philadelphia: Linguistic Data Consortium, 2011.",
        )

    def _split_generators(self, dl_manager):
        loaded_files = dl_manager.download_and_extract(self.config.data_files)
        return [
            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": loaded_files["train"]}),
            datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepath": loaded_files["test"]}),
            datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={"filepath": loaded_files["val"]})
        ]

    def _generate_examples(self, filepath):
        logger.info("generating examples from = %s", filepath)
        with open(filepath, encoding="cp1252") as f:
            data = csv.reader(f, delimiter="\t")
            ids = list()
            tokens = list()
            pos_tags = list()
            for id_, line in enumerate(data):
                #print(line)
                if len(line) == 1:
                    if tokens:
                        yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}
                        ids = list()
                        tokens = list()
                        pos_tags = list()
                else:
                    ids.append(line[0])
                    tokens.append(line[1])
                    pos_tags.append(line[2])
            # last example
            yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}

def main():
    dataset = datasets.load_dataset(
        "data_loading.py", data_files={
            "train": "train.tsv",
            "test": "test.tsv",
            "val": "val.tsv"
        }
    )

    #print(dataset)

if __name__=="__main__":
    main()
```

 Hi @itsLuisa, thanks for sharing the Traceback.

You are defining the "id" field as a `string` feature:
```python
class Sample(datasets.GeneratorBasedBuilder):
    ...

    def _info(self):
        return datasets.DatasetInfo(
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    # ^^ here
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "pos_tags": datasets.Sequence(datasets.features.ClassLabel(names=[...])),
[...]
```

But in the `_generate_examples`, the "id" field is a list:
```python
ids = list()
```

Changing:
```python
"id": datasets.Value("string"),
```
Into:
```python
"id": datasets.Sequence(datasets.Value("string")),
```

Should fix your issue.

Let me know if this helps!


Error when exploring `arabic_speech_corpus`
 Navigate to https://huggingface.co/datasets/viewer/?dataset=arabic_speech_corpus

Error:
```
ImportError: To be able to use this dataset, you need to install the following dependencies['soundfile'] using 'pip install soundfile' for instance'
Traceback:
File "/home/sasha/.local/share/virtualenvs/lib-ogGKnCK_/lib/python3.7/site-packages/streamlit/script_runner.py", line 332, in _run_script
    exec(code, module.__dict__)
File "/home/sasha/nlp-viewer/run.py", line 233, in <module>
    configs = get_confs(option)
File "/home/sasha/.local/share/virtualenvs/lib-ogGKnCK_/lib/python3.7/site-packages/streamlit/caching.py", line 604, in wrapped_func
    return get_or_create_cached_value()
File "/home/sasha/.local/share/virtualenvs/lib-ogGKnCK_/lib/python3.7/site-packages/streamlit/caching.py", line 588, in get_or_create_cached_value
    return_value = func(*args, **kwargs)
File "/home/sasha/nlp-viewer/run.py", line 145, in get_confs
    module_path = nlp.load.prepare_module(path, dataset=True
File "/home/sasha/.local/share/virtualenvs/lib-ogGKnCK_/lib/python3.7/site-packages/datasets/load.py", line 342, in prepare_module
    f"To be able to use this {module_type}, you need to install the following dependencies"
```
 Actually soundfile is not a dependency of this dataset.
The error comes from a bug that was fixed in this commit: https://github.com/huggingface/datasets/pull/1767/commits/c304e63629f4453367de2fd42883a78768055532
Basically the library used to consider the `import soundfile` in the docstring as a dependency, while it's just here as a code example.

Updating the viewer to the latest version of `datasets` should fix this issue


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 @lhoestq  I really appreciate if you could help me providiing processed datasets, I do not really have access to enough resources to run the apache-beam and need to run the codes on these datasets. Only en/de/fr currently works, but I need all the languages more or less. thanks


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 Hi @dorost1234, I think I can help you a little. I’ve processed some Wikipedia datasets (Spanish inclusive) using the HF/datasets library during recent research.

@lhoestq Could you help me to upload these preprocessed datasets to Huggingface's repositories? To be more precise, I've built datasets from the following languages using the 20201201 dumps: Spanish, Portuguese, Russian, French, Japanese, Chinese, and Turkish. Process these datasets have high costs that most of the community can't afford. I think these preprocessed datasets I have could be helpful for someone without access to high-resource machines to process Wikipedia's dumps like @dorost1234


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 Thank you so much @jonatasgrosman , I greatly appreciate your help with them.
Yes, I unfortunately does not have access to a good resource and need it for my
research. I greatly appreciate @lhoestq  your help with uploading the processed datasets in huggingface datasets. This would be really helpful for some users like me with not access to high-memory GPU resources.

thank you both so much again.

On Sat, Mar 6, 2021 at 12:55 AM Jonatas Grosman <notifications@github.com>
wrote:

> Hi @dorost1234 <https://github.com/dorost1234>, I think I can help you a
> little. I’ve processed some Wikipedia datasets (Spanish inclusive) using
> the HF/datasets library during recent research.
>
> @lhoestq <https://github.com/lhoestq> Could you help me to upload these
> preprocessed datasets to Huggingface's repositories? To be more precise,
> I've built datasets from the following languages using the 20201201 dumps:
> Spanish, Portuguese, Russian, French, Japanese, Chinese, and Turkish.
> Process these datasets have high costs that most of the community can't
> afford. I think these preprocessed datasets I have could be helpful for
> someone without access to high-resource machines to process Wikipedia's
> dumps like @dorost1234 <https://github.com/dorost1234>
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/huggingface/datasets/issues/1994#issuecomment-791798195>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AS37NMWMK5GFJFU3ACCJFUDTCFVNZANCNFSM4YUZIF4A>
> .
>


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 Hi @dorost1234, so sorry, but looking at my files here, I figure out that I've preprocessed files using the HF/datasets for all the languages previously listed by me (Portuguese, Russian, French, Japanese, Chinese, and Turkish) except the Spanish (on my tests I've used the [wikicorpus](https://www.cs.upc.edu/~nlp/wikicorpus/) instead).

Only with the Spanish Wikipedia's dump, I had the same `KeyError: '000nbsp'` problem already reported here https://github.com/huggingface/datasets/issues/577

So nowadays, even with access to a high resource machine, you couldn't be able to get Wikipedia's Spanish data using the HF/datasets :(


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 Thanks a lot for the information and help. This would be great to have
these datasets.
@lhoestq <https://github.com/lhoestq>  Do you know a way I could get
smaller amount of these data like 1 GBtype of each language to deal with
computatioanl requirements? thanks

On Sat, Mar 6, 2021 at 5:36 PM Jonatas Grosman <notifications@github.com>
wrote:

> Hi @dorost1234 <https://github.com/dorost1234>, so sorry, but looking at
> my files here, I figure out that I've preprocessed files using the
> HF/datasets for all the languages previously listed by me (Portuguese,
> Russian, French, Japanese, Chinese, and Turkish) except the Spanish (on my
> tests I've used the wikicorpus <https://www.cs.upc.edu/~nlp/wikicorpus/>
> instead).
>
> Only with the Spanish Wikipedia's dump, I had the same KeyError: '000nbsp'
> problem already reported here #577
> <https://github.com/huggingface/datasets/issues/577>
>
> So nowadays, even with access to a high resource machine, you couldn't be
> able to get Wikipedia's Spanish data using the HF/datasets :(
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/huggingface/datasets/issues/1994#issuecomment-791985546>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AS37NMWMO7WOHWLOROPD6Q3TCJKXPANCNFSM4YUZIF4A>
> .
>


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 Hi ! As mentioned above the Spanish configuration have parsing issues from `mwparserfromhell`. I haven't tested with the latest `mwparserfromhell` >=0.6 though. Which version of `mwparserfromhell` are you using ?

> @lhoestq Could you help me to upload these preprocessed datasets to Huggingface's repositories? To be more precise, I've built datasets from the following languages using the 20201201 dumps: Spanish, Portuguese, Russian, French, Japanese, Chinese, and Turkish. Process these datasets have high costs that most of the community can't afford. I think these preprocessed datasets I have could be helpful for someone without access to high-resource machines to process Wikipedia's dumps like @dorost1234

That would be awesome ! Feel free to ping me on slack so we can put the processed wikipedia files on google storage with the other ones we've already preprocessed.

> Do you know a way I could get smaller amount of these data like 1 GBtype of each language to deal with computatioanl requirements? thanks

I'd suggest to copy the [wikipedia.py](https://github.com/huggingface/datasets/blob/master/datasets/wikipedia/wikipedia.py) to a new script `custom_wikipedia.py` and modify it to only download and process only a subset of the raw data files.
You can for example replace [this line](https://github.com/huggingface/datasets/blob/64e59fc45ca2134218b3e42e83fddddbe840ff74/datasets/wikipedia/wikipedia.py#L446) by:
```python
            if total_bytes >= (1 << 30):  # stop if the total amount of data is >= 1GB
                break
            else:
                xml_urls.append(_base_url(lang) + fname)
```

Then you can load your custom wikipedia dataset with
```python
load_dataset("path/to/my/custom_wikipedia.py", f"{date}.{language}")
```


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 Hi @lhoestq!

> Hi ! As mentioned above the Spanish configuration have parsing issues from mwparserfromhell. I haven't tested with the latest mwparserfromhell >=0.6 though. Which version of mwparserfromhell are you using ?

I'm using the latest mwparserfromhell version (0.6)

> That would be awesome ! Feel free to ping me on slack so we can put the processed wikipedia files on google storage with the other ones we've already preprocessed.

I'll ping you there 👍


not being able to get wikipedia es language
 Hi
I am trying to run a code with wikipedia of config 20200501.es, getting:

Traceback (most recent call last):
  File "run_mlm_t5.py", line 608, in <module>
    main()
  File "run_mlm_t5.py", line 359, in main
    datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/load.py", line 612, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 527, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/dara/libs/anaconda3/envs/success432/lib/python3.7/site-packages/datasets-1.2.1-py3.7.egg/datasets/builder.py", line 1050, in _download_and_prepare
    "\n\t`{}`".format(usage_example)
datasets.builder.MissingBeamOptions: Trying to generate a dataset using Apache Beam, yet no Beam Runner or PipelineOptions() has been provided in `load_dataset` or in the builder arguments. For big datasets it has to run on large-scale data processing tools like Dataflow, Spark, etc. More information about Apache Beam runners at https://beam.apache.org/documentation/runners/capability-matrix/
If you really want to run it locally because you feel like the Dataset is small enough, you can use the local beam runner called `DirectRunner` (you may run out of memory).
Example of usage:
	`load_dataset('wikipedia', '20200501.es', beam_runner='DirectRunner')`

thanks @lhoestq  for any suggestion/help
 Thank you so much @jonatasgrosman  and @lhoestq  this would be a great help. I am really thankful to you both and to wonderful Huggingface dataset library allowing us to train models at scale.


How to load a dataset with load_from disk and save it again after doing transformations without changing the original?
 I am using the latest datasets library.  In my work, I first use **load_from_disk** to load a data set that contains 3.8Gb information. Then during my training process, I update that dataset object and add new elements and save it in a different place.

When I save the dataset with **save_to_disk**, the original dataset which is already in the disk also gets updated. I do not want to update it.  How to prevent from this?

 Hi ! That looks like a bug, can you provide some code so that we can reproduce ?
It's not supposed to update the original dataset


How to load a dataset with load_from disk and save it again after doing transformations without changing the original?
 I am using the latest datasets library.  In my work, I first use **load_from_disk** to load a data set that contains 3.8Gb information. Then during my training process, I update that dataset object and add new elements and save it in a different place.

When I save the dataset with **save_to_disk**, the original dataset which is already in the disk also gets updated. I do not want to update it.  How to prevent from this?

 Hi, I experimented with RAG.

Actually, you can run the [use_own_knowldge_dataset.py](https://github.com/shamanez/transformers/blob/rag-end-to-end-retrieval/examples/research_projects/rag/use_own_knowledge_dataset.py#L80). In the 80 you can save the dataset object to the disk with save_to_disk. Then in order to compute the embeddings in this use **load_from_disk**.

Then finally save it. You can see the original dataset object (CSV after splitting also will be changed)

One more thing- when I save the dataset object with **save_to_disk** it name the arrow file with cache.... rather than using dataset. arrow.  Can you add a variable that we can feed a name to save_to_disk function?


How to load a dataset with load_from disk and save it again after doing transformations without changing the original?
 I am using the latest datasets library.  In my work, I first use **load_from_disk** to load a data set that contains 3.8Gb information. Then during my training process, I update that dataset object and add new elements and save it in a different place.

When I save the dataset with **save_to_disk**, the original dataset which is already in the disk also gets updated. I do not want to update it.  How to prevent from this?

 @lhoestq   I also found that cache in tmp directory gets updated after transformations. This is really problematic when using datasets interactively. Let's say we use the shards function to a dataset loaded with csv, atm when we do transformations to shards and combine them it updates the original csv cache.


How to load a dataset with load_from disk and save it again after doing transformations without changing the original?
 I am using the latest datasets library.  In my work, I first use **load_from_disk** to load a data set that contains 3.8Gb information. Then during my training process, I update that dataset object and add new elements and save it in a different place.

When I save the dataset with **save_to_disk**, the original dataset which is already in the disk also gets updated. I do not want to update it.  How to prevent from this?

 I plan to update the save_to_disk method in #2025 so I can make sure the new save_to_disk doesn't corrupt your cache files.
But from your last message it looks like save_to_disk isn't the root cause right ?


How to load a dataset with load_from disk and save it again after doing transformations without changing the original?
 I am using the latest datasets library.  In my work, I first use **load_from_disk** to load a data set that contains 3.8Gb information. Then during my training process, I update that dataset object and add new elements and save it in a different place.

When I save the dataset with **save_to_disk**, the original dataset which is already in the disk also gets updated. I do not want to update it.  How to prevent from this?

 ok, one more thing. When we use save_to_disk there are two files other than .arrow. dataset_info.json and state.json. Sometimes most of the fields in the dataset_infor.json are null, especially when saving dataset objects. Anyways I think load_from_disk uses the arrow files mentioned in state.json right?


How to load a dataset with load_from disk and save it again after doing transformations without changing the original?
 I am using the latest datasets library.  In my work, I first use **load_from_disk** to load a data set that contains 3.8Gb information. Then during my training process, I update that dataset object and add new elements and save it in a different place.

When I save the dataset with **save_to_disk**, the original dataset which is already in the disk also gets updated. I do not want to update it.  How to prevent from this?

 Perfect.  For now, I am loading the dataset from CSV in my interactive process and will wait until you make the PR!


`datasets.map` multi processing much slower than single processing
 Hi, thank you for the great library.

I've been using datasets to pretrain language models, and it often involves datasets as large as ~70G.
My data preparation step is roughly two steps: `load_dataset` which splits corpora into a table of sentences, and `map` converts a sentence into a list of integers, using a tokenizer.

I noticed that `map` function with `num_proc=mp.cpu_count() //2` takes more than 20 hours to finish the job where as `num_proc=1` gets the job done in about 5 hours. The machine I used has 40 cores, with 126G of RAM. There were no other jobs when `map` function was running.

What could be the reason? I would be happy to provide information necessary to spot the reason.

p.s. I was experiencing the imbalance issue mentioned in [here](https://github.com/huggingface/datasets/issues/610#issuecomment-705177036) when I was using multi processing.
p.s.2 When I run `map` with `num_proc=1`, I see one tqdm bar but all the cores are working. When `num_proc=20`, only 20 cores work.
![Screen Shot 2021-03-05 at 11 04 59](https://user-images.githubusercontent.com/29157715/110056895-ef6cf000-7da2-11eb-8307-6698e9fb1ad4.png)

 Hi @hwijeen, you might want to look at issues #1796 and #1949. I think it could be something related to the I/O operations being performed.


`datasets.map` multi processing much slower than single processing
 Hi, thank you for the great library.

I've been using datasets to pretrain language models, and it often involves datasets as large as ~70G.
My data preparation step is roughly two steps: `load_dataset` which splits corpora into a table of sentences, and `map` converts a sentence into a list of integers, using a tokenizer.

I noticed that `map` function with `num_proc=mp.cpu_count() //2` takes more than 20 hours to finish the job where as `num_proc=1` gets the job done in about 5 hours. The machine I used has 40 cores, with 126G of RAM. There were no other jobs when `map` function was running.

What could be the reason? I would be happy to provide information necessary to spot the reason.

p.s. I was experiencing the imbalance issue mentioned in [here](https://github.com/huggingface/datasets/issues/610#issuecomment-705177036) when I was using multi processing.
p.s.2 When I run `map` with `num_proc=1`, I see one tqdm bar but all the cores are working. When `num_proc=20`, only 20 cores work.
![Screen Shot 2021-03-05 at 11 04 59](https://user-images.githubusercontent.com/29157715/110056895-ef6cf000-7da2-11eb-8307-6698e9fb1ad4.png)

 I see that many people are experiencing the same issue. Is this problem considered an "official" bug that is worth a closer look? @lhoestq


`datasets.map` multi processing much slower than single processing
 Hi, thank you for the great library.

I've been using datasets to pretrain language models, and it often involves datasets as large as ~70G.
My data preparation step is roughly two steps: `load_dataset` which splits corpora into a table of sentences, and `map` converts a sentence into a list of integers, using a tokenizer.

I noticed that `map` function with `num_proc=mp.cpu_count() //2` takes more than 20 hours to finish the job where as `num_proc=1` gets the job done in about 5 hours. The machine I used has 40 cores, with 126G of RAM. There were no other jobs when `map` function was running.

What could be the reason? I would be happy to provide information necessary to spot the reason.

p.s. I was experiencing the imbalance issue mentioned in [here](https://github.com/huggingface/datasets/issues/610#issuecomment-705177036) when I was using multi processing.
p.s.2 When I run `map` with `num_proc=1`, I see one tqdm bar but all the cores are working. When `num_proc=20`, only 20 cores work.
![Screen Shot 2021-03-05 at 11 04 59](https://user-images.githubusercontent.com/29157715/110056895-ef6cf000-7da2-11eb-8307-6698e9fb1ad4.png)

 Yes this is an official bug. On my side I haven't managed to reproduce it but @theo-m has. We'll investigate this !


`datasets.map` multi processing much slower than single processing
 Hi, thank you for the great library.

I've been using datasets to pretrain language models, and it often involves datasets as large as ~70G.
My data preparation step is roughly two steps: `load_dataset` which splits corpora into a table of sentences, and `map` converts a sentence into a list of integers, using a tokenizer.

I noticed that `map` function with `num_proc=mp.cpu_count() //2` takes more than 20 hours to finish the job where as `num_proc=1` gets the job done in about 5 hours. The machine I used has 40 cores, with 126G of RAM. There were no other jobs when `map` function was running.

What could be the reason? I would be happy to provide information necessary to spot the reason.

p.s. I was experiencing the imbalance issue mentioned in [here](https://github.com/huggingface/datasets/issues/610#issuecomment-705177036) when I was using multi processing.
p.s.2 When I run `map` with `num_proc=1`, I see one tqdm bar but all the cores are working. When `num_proc=20`, only 20 cores work.
![Screen Shot 2021-03-05 at 11 04 59](https://user-images.githubusercontent.com/29157715/110056895-ef6cf000-7da2-11eb-8307-6698e9fb1ad4.png)

 Thank you for the reply! I would be happy to follow the discussions related to the issue.
If you do not mind, could you also give a little more explanation on my p.s.2? I am having a hard time figuring out why the single processing `map` uses all of my cores.
@lhoestq @theo-m


`datasets.map` multi processing much slower than single processing
 Hi, thank you for the great library.

I've been using datasets to pretrain language models, and it often involves datasets as large as ~70G.
My data preparation step is roughly two steps: `load_dataset` which splits corpora into a table of sentences, and `map` converts a sentence into a list of integers, using a tokenizer.

I noticed that `map` function with `num_proc=mp.cpu_count() //2` takes more than 20 hours to finish the job where as `num_proc=1` gets the job done in about 5 hours. The machine I used has 40 cores, with 126G of RAM. There were no other jobs when `map` function was running.

What could be the reason? I would be happy to provide information necessary to spot the reason.

p.s. I was experiencing the imbalance issue mentioned in [here](https://github.com/huggingface/datasets/issues/610#issuecomment-705177036) when I was using multi processing.
p.s.2 When I run `map` with `num_proc=1`, I see one tqdm bar but all the cores are working. When `num_proc=20`, only 20 cores work.
![Screen Shot 2021-03-05 at 11 04 59](https://user-images.githubusercontent.com/29157715/110056895-ef6cf000-7da2-11eb-8307-6698e9fb1ad4.png)

 Regarding your ps2: It depends what function you pass to `map`.
For example, fast tokenizers from `transformers` in Rust tokenize texts and parallelize the tokenization over all the cores.


`datasets.map` multi processing much slower than single processing
 Hi, thank you for the great library.

I've been using datasets to pretrain language models, and it often involves datasets as large as ~70G.
My data preparation step is roughly two steps: `load_dataset` which splits corpora into a table of sentences, and `map` converts a sentence into a list of integers, using a tokenizer.

I noticed that `map` function with `num_proc=mp.cpu_count() //2` takes more than 20 hours to finish the job where as `num_proc=1` gets the job done in about 5 hours. The machine I used has 40 cores, with 126G of RAM. There were no other jobs when `map` function was running.

What could be the reason? I would be happy to provide information necessary to spot the reason.

p.s. I was experiencing the imbalance issue mentioned in [here](https://github.com/huggingface/datasets/issues/610#issuecomment-705177036) when I was using multi processing.
p.s.2 When I run `map` with `num_proc=1`, I see one tqdm bar but all the cores are working. When `num_proc=20`, only 20 cores work.
![Screen Shot 2021-03-05 at 11 04 59](https://user-images.githubusercontent.com/29157715/110056895-ef6cf000-7da2-11eb-8307-6698e9fb1ad4.png)

 I am still experiencing this issue with datasets 1.9.0..
Has there been a further investigation?
<img width="442" alt="image" src="https://user-images.githubusercontent.com/29157715/126143387-8b5ddca2-a896-4e18-abf7-4fbf62a48b41.png">


OSError: Memory mapping file failed: Cannot allocate memory
 Hi,
I am trying to run a code with a wikipedia dataset, here is the command to reproduce the error. You can find the codes for run_mlm.py in huggingface repo here: https://github.com/huggingface/transformers/blob/v4.3.2/examples/language-modeling/run_mlm.py
```
python run_mlm.py --model_name_or_path bert-base-multilingual-cased --dataset_name wikipedia --dataset_config_name 20200501.en --do_train --do_eval --output_dir /dara/test  --max_seq_length 128
```

I am using transformer version: 4.3.2

But I got memory erorr using this dataset, is there a way I could save on memory with dataset library with wikipedia dataset?
Specially I need to train a model with multiple of wikipedia datasets concatenated. thank you very much @lhoestq  for your help and suggestions:

```
  File "run_mlm.py", line 441, in <module>
    main()
  File "run_mlm.py", line 233, in main
    split=f"train[{data_args.validation_split_percentage}%:]",
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/load.py", line 750, in load_dataset
    ds = builder_instance.as_dataset(split=split, ignore_verifications=ignore_verifications, in_memory=keep_in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 740, in as_dataset
    map_tuple=True,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/utils/py_utils.py", line 225, in map_nested
    return function(data_struct)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 757, in _build_single_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 829, in _as_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 215, in read
    return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 236, in read_files
    pa_table = self._read_files(files, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 171, in _read_files
    pa_table: pa.Table = self._get_dataset_from_filename(f_dict, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 302, in _get_dataset_from_filename
    pa_table = ArrowReader.read_table(filename, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 322, in read_table
    stream = stream_from(filename)
  File "pyarrow/io.pxi", line 782, in pyarrow.lib.memory_map
  File "pyarrow/io.pxi", line 743, in pyarrow.lib.MemoryMappedFile._open
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
OSError: Memory mapping file failed: Cannot allocate memory
```

 Do you think this is trying to bring the dataset into memory and if I can avoid it to save on memory so it only brings a batch into memory? @lhoestq  thank you


OSError: Memory mapping file failed: Cannot allocate memory
 Hi,
I am trying to run a code with a wikipedia dataset, here is the command to reproduce the error. You can find the codes for run_mlm.py in huggingface repo here: https://github.com/huggingface/transformers/blob/v4.3.2/examples/language-modeling/run_mlm.py
```
python run_mlm.py --model_name_or_path bert-base-multilingual-cased --dataset_name wikipedia --dataset_config_name 20200501.en --do_train --do_eval --output_dir /dara/test  --max_seq_length 128
```

I am using transformer version: 4.3.2

But I got memory erorr using this dataset, is there a way I could save on memory with dataset library with wikipedia dataset?
Specially I need to train a model with multiple of wikipedia datasets concatenated. thank you very much @lhoestq  for your help and suggestions:

```
  File "run_mlm.py", line 441, in <module>
    main()
  File "run_mlm.py", line 233, in main
    split=f"train[{data_args.validation_split_percentage}%:]",
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/load.py", line 750, in load_dataset
    ds = builder_instance.as_dataset(split=split, ignore_verifications=ignore_verifications, in_memory=keep_in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 740, in as_dataset
    map_tuple=True,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/utils/py_utils.py", line 225, in map_nested
    return function(data_struct)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 757, in _build_single_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 829, in _as_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 215, in read
    return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 236, in read_files
    pa_table = self._read_files(files, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 171, in _read_files
    pa_table: pa.Table = self._get_dataset_from_filename(f_dict, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 302, in _get_dataset_from_filename
    pa_table = ArrowReader.read_table(filename, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 322, in read_table
    stream = stream_from(filename)
  File "pyarrow/io.pxi", line 782, in pyarrow.lib.memory_map
  File "pyarrow/io.pxi", line 743, in pyarrow.lib.MemoryMappedFile._open
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
OSError: Memory mapping file failed: Cannot allocate memory
```

 It's not trying to bring the dataset into memory.

Actually, it's trying to memory map the dataset file, which is different. It allows to load large dataset files without filling up memory.

What dataset did you use to get this error ?
On what OS are you running ? What's your python and pyarrow version ?


OSError: Memory mapping file failed: Cannot allocate memory
 Hi,
I am trying to run a code with a wikipedia dataset, here is the command to reproduce the error. You can find the codes for run_mlm.py in huggingface repo here: https://github.com/huggingface/transformers/blob/v4.3.2/examples/language-modeling/run_mlm.py
```
python run_mlm.py --model_name_or_path bert-base-multilingual-cased --dataset_name wikipedia --dataset_config_name 20200501.en --do_train --do_eval --output_dir /dara/test  --max_seq_length 128
```

I am using transformer version: 4.3.2

But I got memory erorr using this dataset, is there a way I could save on memory with dataset library with wikipedia dataset?
Specially I need to train a model with multiple of wikipedia datasets concatenated. thank you very much @lhoestq  for your help and suggestions:

```
  File "run_mlm.py", line 441, in <module>
    main()
  File "run_mlm.py", line 233, in main
    split=f"train[{data_args.validation_split_percentage}%:]",
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/load.py", line 750, in load_dataset
    ds = builder_instance.as_dataset(split=split, ignore_verifications=ignore_verifications, in_memory=keep_in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 740, in as_dataset
    map_tuple=True,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/utils/py_utils.py", line 225, in map_nested
    return function(data_struct)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 757, in _build_single_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 829, in _as_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 215, in read
    return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 236, in read_files
    pa_table = self._read_files(files, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 171, in _read_files
    pa_table: pa.Table = self._get_dataset_from_filename(f_dict, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 302, in _get_dataset_from_filename
    pa_table = ArrowReader.read_table(filename, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 322, in read_table
    stream = stream_from(filename)
  File "pyarrow/io.pxi", line 782, in pyarrow.lib.memory_map
  File "pyarrow/io.pxi", line 743, in pyarrow.lib.MemoryMappedFile._open
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
OSError: Memory mapping file failed: Cannot allocate memory
```

 Dear @lhoestq
thank you so much for coming back to me. Please find info below:
1) Dataset name: I used  wikipedia with config 20200501.en
2) I got these pyarrow in my environment:
pyarrow                   2.0.0                     <pip>
pyarrow                   3.0.0                     <pip>

3) python version 3.7.10
4) OS version

lsb_release -a
No LSB modules are available.
Distributor ID:	Debian
Description:	Debian GNU/Linux 10 (buster)
Release:	10
Codename:	buster

Is there a way I could solve the memory issue and if I could run this model, I am using  GeForce GTX 108,
thanks


OSError: Memory mapping file failed: Cannot allocate memory
 Hi,
I am trying to run a code with a wikipedia dataset, here is the command to reproduce the error. You can find the codes for run_mlm.py in huggingface repo here: https://github.com/huggingface/transformers/blob/v4.3.2/examples/language-modeling/run_mlm.py
```
python run_mlm.py --model_name_or_path bert-base-multilingual-cased --dataset_name wikipedia --dataset_config_name 20200501.en --do_train --do_eval --output_dir /dara/test  --max_seq_length 128
```

I am using transformer version: 4.3.2

But I got memory erorr using this dataset, is there a way I could save on memory with dataset library with wikipedia dataset?
Specially I need to train a model with multiple of wikipedia datasets concatenated. thank you very much @lhoestq  for your help and suggestions:

```
  File "run_mlm.py", line 441, in <module>
    main()
  File "run_mlm.py", line 233, in main
    split=f"train[{data_args.validation_split_percentage}%:]",
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/load.py", line 750, in load_dataset
    ds = builder_instance.as_dataset(split=split, ignore_verifications=ignore_verifications, in_memory=keep_in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 740, in as_dataset
    map_tuple=True,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/utils/py_utils.py", line 225, in map_nested
    return function(data_struct)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 757, in _build_single_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 829, in _as_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 215, in read
    return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 236, in read_files
    pa_table = self._read_files(files, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 171, in _read_files
    pa_table: pa.Table = self._get_dataset_from_filename(f_dict, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 302, in _get_dataset_from_filename
    pa_table = ArrowReader.read_table(filename, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 322, in read_table
    stream = stream_from(filename)
  File "pyarrow/io.pxi", line 782, in pyarrow.lib.memory_map
  File "pyarrow/io.pxi", line 743, in pyarrow.lib.MemoryMappedFile._open
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
OSError: Memory mapping file failed: Cannot allocate memory
```

 I noticed that the error happens when loading the validation dataset.
What value of `data_args.validation_split_percentage` did you use ?


OSError: Memory mapping file failed: Cannot allocate memory
 Hi,
I am trying to run a code with a wikipedia dataset, here is the command to reproduce the error. You can find the codes for run_mlm.py in huggingface repo here: https://github.com/huggingface/transformers/blob/v4.3.2/examples/language-modeling/run_mlm.py
```
python run_mlm.py --model_name_or_path bert-base-multilingual-cased --dataset_name wikipedia --dataset_config_name 20200501.en --do_train --do_eval --output_dir /dara/test  --max_seq_length 128
```

I am using transformer version: 4.3.2

But I got memory erorr using this dataset, is there a way I could save on memory with dataset library with wikipedia dataset?
Specially I need to train a model with multiple of wikipedia datasets concatenated. thank you very much @lhoestq  for your help and suggestions:

```
  File "run_mlm.py", line 441, in <module>
    main()
  File "run_mlm.py", line 233, in main
    split=f"train[{data_args.validation_split_percentage}%:]",
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/load.py", line 750, in load_dataset
    ds = builder_instance.as_dataset(split=split, ignore_verifications=ignore_verifications, in_memory=keep_in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 740, in as_dataset
    map_tuple=True,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/utils/py_utils.py", line 225, in map_nested
    return function(data_struct)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 757, in _build_single_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 829, in _as_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 215, in read
    return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 236, in read_files
    pa_table = self._read_files(files, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 171, in _read_files
    pa_table: pa.Table = self._get_dataset_from_filename(f_dict, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 302, in _get_dataset_from_filename
    pa_table = ArrowReader.read_table(filename, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 322, in read_table
    stream = stream_from(filename)
  File "pyarrow/io.pxi", line 782, in pyarrow.lib.memory_map
  File "pyarrow/io.pxi", line 743, in pyarrow.lib.MemoryMappedFile._open
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
OSError: Memory mapping file failed: Cannot allocate memory
```

 Dear @lhoestq

thank you very much for the very sharp observation, indeed, this happens there, I use the default value of 5, I basically plan to subsample a part of the large  dataset and choose it as validation set. Do you think this is bringing the data into memory during subsampling? Is there a way I could avoid this?

Thank you very much for the great help.

On Mon, Mar 8, 2021 at 11:28 AM Quentin Lhoest ***@***.***>
wrote:

> I noticed that the error happens when loading the validation dataset.
> What value of data_args.validation_split_percentage did you use ?
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/huggingface/datasets/issues/1990#issuecomment-792655644>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AS37NMS337ZUJ7HGGVVCCR3TCSREFANCNFSM4YTYAQ2A>
> .
>


OSError: Memory mapping file failed: Cannot allocate memory
 Hi,
I am trying to run a code with a wikipedia dataset, here is the command to reproduce the error. You can find the codes for run_mlm.py in huggingface repo here: https://github.com/huggingface/transformers/blob/v4.3.2/examples/language-modeling/run_mlm.py
```
python run_mlm.py --model_name_or_path bert-base-multilingual-cased --dataset_name wikipedia --dataset_config_name 20200501.en --do_train --do_eval --output_dir /dara/test  --max_seq_length 128
```

I am using transformer version: 4.3.2

But I got memory erorr using this dataset, is there a way I could save on memory with dataset library with wikipedia dataset?
Specially I need to train a model with multiple of wikipedia datasets concatenated. thank you very much @lhoestq  for your help and suggestions:

```
  File "run_mlm.py", line 441, in <module>
    main()
  File "run_mlm.py", line 233, in main
    split=f"train[{data_args.validation_split_percentage}%:]",
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/load.py", line 750, in load_dataset
    ds = builder_instance.as_dataset(split=split, ignore_verifications=ignore_verifications, in_memory=keep_in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 740, in as_dataset
    map_tuple=True,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/utils/py_utils.py", line 225, in map_nested
    return function(data_struct)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 757, in _build_single_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/builder.py", line 829, in _as_dataset
    in_memory=in_memory,
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 215, in read
    return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 236, in read_files
    pa_table = self._read_files(files, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 171, in _read_files
    pa_table: pa.Table = self._get_dataset_from_filename(f_dict, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 302, in _get_dataset_from_filename
    pa_table = ArrowReader.read_table(filename, in_memory=in_memory)
  File "/dara/libs/anaconda3/envs/code/lib/python3.7/site-packages/datasets-1.3.0-py3.7.egg/datasets/arrow_reader.py", line 322, in read_table
    stream = stream_from(filename)
  File "pyarrow/io.pxi", line 782, in pyarrow.lib.memory_map
  File "pyarrow/io.pxi", line 743, in pyarrow.lib.MemoryMappedFile._open
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
OSError: Memory mapping file failed: Cannot allocate memory
```

 Methods like `dataset.shard`, `dataset.train_test_split`, `dataset.select` etc. don't bring the dataset in memory.
The only time when samples are brought to memory is when you access elements via `dataset[0]`, `dataset[:10]`, `dataset["my_column_names"]`.

But it's possible that trying to use those methods to build your validation set doesn't fix the issue since, if I understand correctly, the error happens when when the dataset arrow file is opened (just before the 5% percentage is applied).

Did you try to reproduce this issue in a google colab ? This would be super helpful to investigate why this happened.

Also maybe you can try clearing your cache at `~/.cache/huggingface/datasets` and try again. If the arrow file was corrupted somehow, removing it and rebuilding may fix the issue.


Question/problem with dataset labels

Question/problem with dataset labels
 Hi, I'm using a dataset with two labels "nurse" and "not nurse". For whatever reason (that I don't understand), I get an error that I think comes from the datasets package (using csv). Everything works fine if the labels are "nurse" and "surgeon".

This is the trace I get:

```
File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: not nurse
```

Any ideas how to fix this? For now, I'll probably make them numeric.
 It seems that I get parsing errors for various fields in my data. For example now I get this:
```
  File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: https://www.netgalley.com/catalog/book/121872
```


Question/problem with dataset labels

Question/problem with dataset labels
 Hi, I'm using a dataset with two labels "nurse" and "not nurse". For whatever reason (that I don't understand), I get an error that I think comes from the datasets package (using csv). Everything works fine if the labels are "nurse" and "surgeon".

This is the trace I get:

```
File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: not nurse
```

Any ideas how to fix this? For now, I'll probably make them numeric.
 Not sure if this helps, this is how I load my files (as in the sample scripts on transformers):

```
    if data_args.train_file.endswith(".csv"):
        # Loading a dataset from local csv files
        datasets = load_dataset("csv", data_files=data_files)
```


Question/problem with dataset labels

Question/problem with dataset labels
 Hi, I'm using a dataset with two labels "nurse" and "not nurse". For whatever reason (that I don't understand), I get an error that I think comes from the datasets package (using csv). Everything works fine if the labels are "nurse" and "surgeon".

This is the trace I get:

```
File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: not nurse
```

Any ideas how to fix this? For now, I'll probably make them numeric.
 Since this worked out of the box in a few examples before, I wonder if it's some quoting issue or something else.


Question/problem with dataset labels

Question/problem with dataset labels
 Hi, I'm using a dataset with two labels "nurse" and "not nurse". For whatever reason (that I don't understand), I get an error that I think comes from the datasets package (using csv). Everything works fine if the labels are "nurse" and "surgeon".

This is the trace I get:

```
File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: not nurse
```

Any ideas how to fix this? For now, I'll probably make them numeric.
 Hi @ioana-blue,
Can you share a sample from your .csv? A dummy where you get this error will also help.

I tried this csv:
```csv
feature,label
1.2,not nurse
1.3,nurse
1.5,surgeon
```
and the following snippet:
```python
from datasets import load_dataset

d = load_dataset("csv",data_files=['test.csv'])

print(d)
print(d['train']['label'])
```
and this works perfectly fine for me:
```sh
DatasetDict({
    train: Dataset({
        features: ['feature', 'label'],
        num_rows: 3
    })
})
['not nurse', 'nurse', 'surgeon']
```
I'm sure your csv is more complicated than this one. But it is hard to tell where the issue might be without looking at a sample.


Question/problem with dataset labels

Question/problem with dataset labels
 Hi, I'm using a dataset with two labels "nurse" and "not nurse". For whatever reason (that I don't understand), I get an error that I think comes from the datasets package (using csv). Everything works fine if the labels are "nurse" and "surgeon".

This is the trace I get:

```
File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: not nurse
```

Any ideas how to fix this? For now, I'll probably make them numeric.
 I've had versions where it worked fain. For this dataset, I had all kind of parsing issues that I couldn't understand. What I ended up doing is strip all the columns that I didn't need and also make the label 0/1.

I think one line that may have caused a problem was the csv version of this:

```crawl-data/CC-MAIN-2017-47/segments/1510934806225.78/wet/CC-MAIN-20171120203833-20171120223833-00571.warc.wet.gz        Rose Blakey is an aspiring journalist. She is desperate to escape the from the small Australian town in which she lives. Rejection after rejection mean she is stuck in what she sees as a dead-end waitressing job. ^M ('Rose', '', 'Blakey')  journalist      F       38      journalist      https://www.netgalley.com/catalog/book/121872   _ is desperate to escape the from the small Australian town in which _ lives. Rejection after rejection mean _ is stuck in what _ sees as a dead-end waitressing job.    She is desperate to escape the from the small Australian town in which she lives. Rejection after rejection mean she is stuck in what she sees as a dead-end waitressing job.```

The error I got in this case is this one: https://github.com/huggingface/datasets/issues/1989#issuecomment-790842771

Note, this line was part of a much larger file and until this line I guess it was working fine.


Question/problem with dataset labels

Question/problem with dataset labels
 Hi, I'm using a dataset with two labels "nurse" and "not nurse". For whatever reason (that I don't understand), I get an error that I think comes from the datasets package (using csv). Everything works fine if the labels are "nurse" and "surgeon".

This is the trace I get:

```
File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: not nurse
```

Any ideas how to fix this? For now, I'll probably make them numeric.
 Hi @ioana-blue,

What is the separator you're using for the csv? I see there are only two commas in the given line, but they don't seem like appropriate points. Also, is this a string part of one line, or an entire line? There should also be a label, right?


Question/problem with dataset labels

Question/problem with dataset labels
 Hi, I'm using a dataset with two labels "nurse" and "not nurse". For whatever reason (that I don't understand), I get an error that I think comes from the datasets package (using csv). Everything works fine if the labels are "nurse" and "surgeon".

This is the trace I get:

```
File "../../../models/tr-4.3.2/run_puppets.py", line 523, in <module>
    main()
  File "../../../models/tr-4.3.2/run_puppets.py", line 249, in main
    datasets = load_dataset("csv", data_files=data_files)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 572, in download_and_prepare
    self._download_and_prepare(
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 650, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/builder.py", line 1028, in _prepare_split
    writer.write_table(table)
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/datasets/arrow_writer.py", line 292, in write_table
    pa_table = pa_table.cast(self._schema)
  File "pyarrow/table.pxi", line 1311, in pyarrow.lib.Table.cast
  File "pyarrow/table.pxi", line 265, in pyarrow.lib.ChunkedArray.cast
  File "/dccstor/redrug_ier/envs/last-tr/lib/python3.8/site-packages/pyarrow/compute.py", line 87, in cast
    return call_function("cast", [arr], options)
  File "pyarrow/_compute.pyx", line 298, in pyarrow._compute.call_function
  File "pyarrow/_compute.pyx", line 192, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: Failed to parse string: not nurse
```

Any ideas how to fix this? For now, I'll probably make them numeric.
 Sorry for the confusion, the sample above was from a tsv that was used to derive the csv. Let me construct the csv again (I had remove it).

This is the line in the csv - this is the whole line:
```crawl-data/CC-MAIN-2017-47/segments/1510934806225.78/wet/CC-MAIN-20171120203833-20171120223833-00571.warc.wet.gz,Rose Blakey is an aspiring journalist. She is desperate to escape the from the small Australian town in which she lives. Rejection after rejection mean she is stuck in what she sees as a dead,"('Rose', '', 'Blakey')",journalist,F,38,journalist,https://www.netgalley.com/catalog/book/121872,_ is desperate to escape the from the small Australian town in which _ lives. Rejection after rejection mean _ is stuck in what _ sees as a dead-end waitressing job., She is desperate to escape the from the small Australian town in which she lives. Rejection after rejection mean she is stuck in what she sees as a dead-end waitressing job.```