Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / computer-vision-course /pr_397 /en /unit10 /challenges.html

rtrm

about 1 month ago

download

raw

22.2 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Challenges and Opportunities Associated With Using Synthetic Data","local":"challenges-and-opportunities-associated-with-using-synthetic-data","sections":[{"title":"Overfitting the Model","local":"overfitting-the-model","sections":[],"depth":2},{"title":"Are there biases in the synthetic data?","local":"are-there-biases-in-the-synthetic-data","sections":[],"depth":2},{"title":"Does the benefit of using synthetic data outweigh the computational cost?","local":"does-the-benefit-of-using-synthetic-data-outweigh-the-computational-cost","sections":[{"title":"What is the perceived quality of the synthetic images?","local":"what-is-the-perceived-quality-of-the-synthetic-images","sections":[],"depth":3}],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2},{"title":"Resources and Further Reading","local":"resources-and-further-reading","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/computer-vision-course/pr_397/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/entry/start.7f209408.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/chunks/scheduler.7bc62968.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/chunks/singletons.b15acae1.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/chunks/paths.11cdc4b4.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/entry/app.32e8338e.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/chunks/index.2f8492b0.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/nodes/0.e37092e8.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/nodes/15.f9113be0.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/chunks/Tip.016f38d9.js">
	<link rel="modulepreload" href="/docs/computer-vision-course/pr_397/en/_app/immutable/chunks/index.514d62da.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Challenges and Opportunities Associated With Using Synthetic Data","local":"challenges-and-opportunities-associated-with-using-synthetic-data","sections":[{"title":"Overfitting the Model","local":"overfitting-the-model","sections":[],"depth":2},{"title":"Are there biases in the synthetic data?","local":"are-there-biases-in-the-synthetic-data","sections":[],"depth":2},{"title":"Does the benefit of using synthetic data outweigh the computational cost?","local":"does-the-benefit-of-using-synthetic-data-outweigh-the-computational-cost","sections":[{"title":"What is the perceived quality of the synthetic images?","local":"what-is-the-perceived-quality-of-the-synthetic-images","sections":[],"depth":3}],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2},{"title":"Resources and Further Reading","local":"resources-and-further-reading","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="challenges-and-opportunities-associated-with-using-synthetic-data" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#challenges-and-opportunities-associated-with-using-synthetic-data"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Challenges and Opportunities Associated With Using Synthetic Data</span></h1> <p data-svelte-h="svelte-u7qs1">Training machine learning models requires vast amounts of data. Synthetic data can help by addressing privacy issues, augmenting limited data, and correcting imbalances in the real data. We have learned how to generate synthetic data using several different methods. Before using synthetic data to train a model, however, there are several important things that need to be considered.</p> <h2 class="relative group"><a id="overfitting-the-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#overfitting-the-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Overfitting the Model</span></h2> <p data-svelte-h="svelte-11btgvk">Overfitting occurs when a machine learning model learns the training data so well that it doesn’t perform well on new, unseen data.
	It’s akin to learning a specific way to solve a problem but then encountering a new situation where the strategy doesn’t work. If the process of generating synthetic data is too simple or there are are overly-consistent patterns, your model might overfit to the limited variations present in the synthetic data. As a very simple example, suppose you trained a model using a synthetic dataset of 25 red circles and 25 blue squares. The model will probably learn to associate circles with the color red and squares with the color blue. This model would likely fail if presented with a red square.</p> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400">Be sure to double check that your dataset doesn't have the following types of
	patterns!</div> <p data-svelte-h="svelte-uz71c9"><em>Overly Consistent Color</em> <img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/synthetic-data-creation-overfit/overfit-color.jpg" alt="consistent-color"></p> <p data-svelte-h="svelte-wi9xvt"><em>Overly Consistent Size</em> <img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/synthetic-data-creation-overfit/overfit-size.jpg" alt="consistent-size"></p> <p data-svelte-h="svelte-wjqm5u"><em>Overly Consistent Background</em> <img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/synthetic-data-creation-overfit/overfit-background.jpg" alt="consistent-background"></p> <p data-svelte-h="svelte-1gvpwq7"><em>Overly Consistent Location</em> <img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/synthetic-data-creation-overfit/overfit-location.jpg" alt="consistent-location"></p> <h2 class="relative group"><a id="are-there-biases-in-the-synthetic-data" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#are-there-biases-in-the-synthetic-data"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Are there biases in the synthetic data?</span></h2> <p data-svelte-h="svelte-1qrtcjb">If the process of generating synthetic data has biases or inaccuracies, your model may unintentionally learn and perpetuate those biases. Beware of the following pitfalls:</p> <p data-svelte-h="svelte-9zsifk"><strong>Limited Diversity</strong></p> <p data-svelte-h="svelte-123qi28">One challenge is that synthetic data may fail to adequately represent the complexity and diversity of the real data. The shape example might seem trivial, but there are lots of situations where failing to account for the wide variety of people, places, animals, or objects will result in a model that doesn’t perform well. For example, suppose you wanted to train a model to monitor the population of an endangered species, such as aye-aye lemurs. If your dataset only contains images of ring-tailed lemurs, the model might struggle to accurately identify aye-aye lemurs in the wild. This limitation could lead to errors in population assessments. The great thing is that if you are mindful of any imbalances in the underlying dataset, you can potentially use synthetic data to de-bias the real data by augmenting with synthetic data from the under-represented class.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Try to make sure your dataset reflects the variety found in the real world!</div> <p data-svelte-h="svelte-1kzx32r"><strong>Nice Variety</strong> <img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/synthetic-data-creation-overfit/good-variety.jpg" alt="nice-variety"></p> <p data-svelte-h="svelte-8t5p2y"><strong>Copying Existing Biases:</strong></p> <p data-svelte-h="svelte-uxf8nr">If the data you used to create the synthetic images already had biases, your model might unintentionally learn and replicate those biases. It’s like copying a friend’s notes without realizing they made a mistake – your computer might end up with the same errors.</p> <h2 class="relative group"><a id="does-the-benefit-of-using-synthetic-data-outweigh-the-computational-cost" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#does-the-benefit-of-using-synthetic-data-outweigh-the-computational-cost"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Does the benefit of using synthetic data outweigh the computational cost?</span></h2> <p data-svelte-h="svelte-10d2zlq">Generating high-quality synthetic data can be computationally expensive. This may pose challenges in terms of both time and resources, especially for complex models or large datasets. As a general rule, generating and using a synthetic dataset only makes sense if it ultimately saves resources (money, time, etc.).</p> <h3 class="relative group"><a id="what-is-the-perceived-quality-of-the-synthetic-images" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-is-the-perceived-quality-of-the-synthetic-images"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What is the perceived quality of the synthetic images?</span></h3> <p data-svelte-h="svelte-2nhqea">Let’s consider the lung images we generated using DCGAN. While some of the images looked pretty realistic, others were not so good. A model trained with the low-quality images might fail to detect pneumonia because they contained noise that isn’t present in the real images. It is also possible that your model might get really good at recognizing patterns in the synthetic data, but those patterns might not existor may be different in the real world.</p> <p data-svelte-h="svelte-153wbhu">A good practice is to evaluate your dataset using a metric such as Frechet Inception Distance (FID), Inception Score (IS), or the Classification Accuracy Score (CAS).</p> <p data-svelte-h="svelte-1xl2ts6"><em>FID:</em></p> <p data-svelte-h="svelte-1sz1dvu">FID uses a pre-trained neural network model, often <a href="https://huggingface.co/docs/timm/models/inception-v4" rel="nofollow">Inception</a>, which is good at recognizing objects in images. The model is used to extract features from both the real and generated images.FID is a measure of how “far” one distribution is from another, taking into account both the mean and covariance of the distributions.</p> <p data-svelte-h="svelte-1p1oso0">A low FID suggests that the feature distributions of real and generated images are similar and the generated images are more likely
	to be realistic.</p> <p data-svelte-h="svelte-1kcxc5n"><em>IS:</em></p> <p data-svelte-h="svelte-8uvvpg">IS uses a pre-trained Inception model to evaluate the quality of generated images produced by generative models, particularly GANs.
	For each generated image, the Inception model assigns a score based on its confidence in recognizing objects within that image. High scores are better and indicate that the Inception model is confident about the content of the image.</p> <p data-svelte-h="svelte-1djui6c"><em>CAS:</em></p> <p data-svelte-h="svelte-19os934">Classification accuracy is another measure of how well your model is performing on the synthetic data. A higher accuracy indicates that the model is effectively capturing the features and patterns of the real images. Low accuracy scores for certain classes may indicate issues with the generation process, such as unrealistic backgrounds, incorrect textures, or inconsistent lighting conditions. You can use CIS to help you identify and address these problems to improve the overall quality of the synthetic dataset.</p> <h2 class="relative group"><a id="conclusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#conclusion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Conclusion</span></h2> <p data-svelte-h="svelte-11to9w1">Even after training your model, it is crucial to continuously monitor its performance in real-world scenarios. If your model encounters new situations or trends that weren’t present in the synthetic data, it might struggle to adapt. Addressing these challenges involves mindful design of the synthetic data generation process and evaluation of the model’s performance on real data. Applying these principles will help to unlock the potential of synthetic data!</p> <h2 class="relative group"><a id="resources-and-further-reading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources-and-further-reading"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Resources and Further Reading</span></h2> <ul data-svelte-h="svelte-18lx4ro"><li><a href="https://arxiv.org/pdf/2303.01268.pdf" rel="nofollow">Analyzing Effects of Fake Training Data on the Performance of Deep Learning Systems</a></li> <li><a href="https://arxiv.org/pdf/2305.10118.pdf" rel="nofollow">Bridging the Gap: Enhancing the Utility of Synthetic Data Via Post-Processing Techniques</a></li> <li><a href="https://arxiv.org/pdf/2303.14126.pdf" rel="nofollow">CIFAKE: Image Classification and Explanable Identification of AI-Generated Synthetic Images</a></li> <li><a href="https://arxiv.org/abs/1905.10887" rel="nofollow">Classification Accuracy Score for Conditional Generative Models</a></li> <li><a href="https://arxiv.org/abs/1706.08500" rel="nofollow">GANs Trained by a Two Time-Scale Update Rule Converge to a Local Nash Equilibrium</a></li> <li><a href="https://arxiv.org/abs/1606.03498" rel="nofollow">Improved Techniques for Training GANs</a></li> <li><a href="https://arxiv.org/pdf/1512.00567v3.pdf" rel="nofollow">Rethinking the Inception Architecture for Computer Vision</a></li> <li><a href="https://github.com/huggingface/community-events/tree/main/huggan/pytorch/metrics" rel="nofollow">Metrics</a></li> <li><a href="https://github.com/mseitzer/pytorch-fid" rel="nofollow">pytorch-fid</a></li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/computer-vision-course/blob/main/chapters/en/unit10/challenges.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1p6gie1 = {
	assets: "/docs/computer-vision-course/pr_397/en",
	base: "/docs/computer-vision-course/pr_397/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/computer-vision-course/pr_397/en/_app/immutable/entry/start.7f209408.js"),
	import("/docs/computer-vision-course/pr_397/en/_app/immutable/entry/app.32e8338e.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 15],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 22.2 kB
Xet hash:: 9bdf847326031b226f228527a9fee290fe02c43642512707487afffe0b4c6054

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.