Buckets:
| """TODO(xquad): Add a description here.""" | |
| import json | |
| import datasets | |
| _CITATION = """\ | |
| """ | |
| _DESCRIPTION = """\ | |
| """ | |
| _URL = "https://huggingface.co/datasets/ai4bharat/IndicQA/resolve/main/data/" | |
| _LANG = ["as", "bn", "gu", "hi", "kn", "ml", "mr", "or", "pa", "ta", "te"] | |
| class IndicqaConfig(datasets.BuilderConfig): | |
| """BuilderConfig for Indicqa""" | |
| def __init__(self, lang, **kwargs): | |
| """ | |
| Args: | |
| lang: string, language for the input text | |
| **kwargs: keyword arguments forwarded to super. | |
| """ | |
| super(IndicqaConfig, self).__init__(version=datasets.Version("1.0.0", ""), **kwargs) | |
| self.lang = lang | |
| class Xquad(datasets.GeneratorBasedBuilder): | |
| """TODO(indicqa): Short description of my dataset.""" | |
| # TODO(indicqa): Set up version. | |
| VERSION = datasets.Version("1.0.0") | |
| BUILDER_CONFIGS = [IndicqaConfig(name=f"indicqa.{lang}", description=_DESCRIPTION, lang=lang) for lang in _LANG] | |
| def _info(self): | |
| # TODO(indicqa): Specifies the datasets.DatasetInfo object | |
| return datasets.DatasetInfo( | |
| # This is the description that will appear on the datasets page. | |
| description=_DESCRIPTION, | |
| # datasets.features.FeatureConnectors | |
| features=datasets.Features( | |
| { | |
| "id": datasets.Value("string"), | |
| "context": datasets.Value("string"), | |
| "question": datasets.Value("string"), | |
| "answers": datasets.features.Sequence( | |
| { | |
| "text": datasets.Value("string"), | |
| "answer_start": datasets.Value("int32"), | |
| } | |
| ), | |
| # These are the features of your dataset like images, labels ... | |
| } | |
| ), | |
| # If there's a common (input, target) tuple from the features, | |
| # specify them here. They'll be used if as_supervised=True in | |
| # builder.as_dataset. | |
| supervised_keys=None, | |
| # Homepage of the dataset for documentation | |
| homepage="", | |
| citation=_CITATION, | |
| ) | |
| def _split_generators(self, dl_manager): | |
| """Returns SplitGenerators.""" | |
| # TODO(indicqa): Downloads the data and defines the splits | |
| # dl_manager is a datasets.download.DownloadManager that can be used to | |
| # download and extract URLs | |
| urls_to_download = {lang: _URL + f"indicqa.{lang}.json" for lang in _LANG} | |
| downloaded_files = dl_manager.download_and_extract(urls_to_download) | |
| return [ | |
| datasets.SplitGenerator( | |
| name=datasets.Split.TEST, | |
| # These kwargs will be passed to _generate_examples | |
| gen_kwargs={"filepath": downloaded_files[self.config.lang]}, | |
| ), | |
| ] | |
| def _generate_examples(self, filepath): | |
| """Yields examples.""" | |
| # TODO(indicqa): Yields (key, example) tuples from the dataset | |
| with open(filepath, encoding="utf-8") as f: | |
| indicqa = json.load(f) | |
| id_ = 0 | |
| for article in indicqa["data"]: | |
| for paragraph in article["paragraphs"]: | |
| context = paragraph["context"].strip() | |
| for qa in paragraph["qas"]: | |
| question = qa["question"].strip() | |
| answer_starts = [answer["answer_start"] for answer in qa["answers"]] | |
| answers = [answer["text"].strip() for answer in qa["answers"]] | |
| # Features currently used are "context", "question", and "answers". | |
| # Others are extracted here for the ease of future expansions. | |
| yield id_, { | |
| "context": context, | |
| "question": question, | |
| "id": qa["id"], | |
| "answers": { | |
| "answer_start": answer_starts, | |
| "text": answers, | |
| }, | |
| } | |
| id_ += 1 |
Xet Storage Details
- Size:
- 4.21 kB
- Xet hash:
- fbc04623cd12d6594a8237c2e2917222d360924101f18bf6bbd229c57b4d3c6d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.