import{s as Ge,n as qe,o as ze}from"../chunks/scheduler.35aab934.js";import{S as Ae,i as Ee,e as o,s as a,c as J,h as Fe,a as l,d as n,b as s,f as Ze,g as W,j as r,k as Re,l as Be,m as i,n as Z,t as R,o as G,p as q}from"../chunks/index.de17b971.js";import{C as Ve}from"../chunks/CopyLLMTxtMenu.48bc5cc2.js";import{C as Qe}from"../chunks/CodeBlock.c1f62de5.js";import{H as ve,E as Ye}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.8fdece89.js";function De(we){let m,E,z,F,h,B,p,V,c,ye=`Image segmentation is dividing an image into meaningful segments. It’s all about creating masks that spotlight each object in the picture.
The intuition behind this task is <em>that it can be viewed as a classification for each pixel of the image</em>.
Segmentation models are the
core models in various industries. They can be found in agriculture and autonomous driving. In the farming world, these models are used
for identifying different land sections and assessing the growth stage of crops. They’re also key players for self-driving cars, where
they are used to identify lanes, sidewalks, and other road users.`,Q,f,be='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/segmentation-example.png" alt="Image segmentation"/>',Y,g,Te=`Different types of segmentations can be applied depending on the context and the intended goal.
The most commonly defined segmentations are the following.`,D,d,Me=`<li><strong>Semantic Segmentation</strong>:  This involves assigning the most probable class to each pixel. For example, in semantic segmentation,
the model does not distinguish between two individual cats but rather focuses on the pixel class. It’s all about classification of
each pixel.</li> <li><strong>Instance Segmentation</strong>: This type involves identifying each instance of an object with a unique mask. It combines aspects of
object detection and segmentation to differentiate between individual objects of the same class.</li> <li><strong>Panoptic Segmentation</strong>: A hybrid approach that combines elements of semantic and instance segmentation. It assigns a class and
an instance to each pixel, effectively integrating the <em>what</em> and <em>where</em> aspects of the image.</li>`,N,u,$e='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/segmentation-types.png" alt="Comparison of segmentation types"/>',X,v,xe=`Choosing the right segmentation type depends on the context and the intended goal. One cool thing is that recent models allow you to achieve the three
segmentation types with a single model. We recommend you to check out this <a href="https://huggingface.co/blog/mask2former" rel="nofollow">article</a>, which introduces Mask2former,
a new model by Meta that achieves the three segmentation types with only a Panoptic dataset.`,K,w,O,y,ke=`You’ve probably heard of U-Net, a popular network used for image segmentation. It’s designed with several convolutional layers and works
in two main phases: the downsampling phase, which compresses the image to understand its features, and the upsampling phase, which expands
the image back to its original size for detailed segmentation.`,ee,b,Le=`Computer vision was once dominated by convolutional models, but it has recently shifted towards the vision transformer approach.
An example is <em><a href="https://arxiv.org/abs/2304.02643" rel="nofollow">Segment anything model (SAM)</a></em> that is a popular prompt based model introduced
in April 2023 by <em>Meta AI Research, FAIR</em>. The model is based on the Vision Transformer (ViT) model and focuses on creating a promptable
(i.e. you can provide words to describe what you would like to segment in the image) segmentation model capable of
zero-shot transfer on new images. The strength of the model comes from its training on the largest dataset available, which includes over
1 billion masks on 11 million images. I recommend you play with <a href="https://segment-anything.com/" rel="nofollow">Meta’s demo</a> on a few images and even
better you can play with the <a href="https://huggingface.co/ybelkada/segment-anything" rel="nofollow">model</a> in transformers.`,te,T,Ce=`Here is an example of how to use the model in transformers. First, we will initialize the <code>mask-generation</code> pipeline.
Then, we will pass the image in pipeline for inference.`,ne,M,ie,$,He='More details on how to use the model can be found in the <a href="https://huggingface.co/docs/transformers/main/en/model_doc/sam" rel="nofollow">documentation</a>.',ae,x,se,k,_e=`You have now seen how to use a segmentation model, but how can you evaluate it? As demonstrated in the previous section, segmentation is
primarily a supervised learning task. This means that the dataset is composed of images and their corresponding masks, which serve as the
ground truth. A few metrics can be used to evaluate your model. The most common ones are:`,oe,L,Ue=`<li><strong>The Intersection over Union (IoU) or Jaccard index</strong> metric is the ratio between the intersection and the union of the predicted mask and the ground truth.
IoU is arguably the most common metric used in segmentation tasks. Its advantage lies in being less sensitive to class imbalance, making
it often a good choice when you begin modeling.</li>`,le,C,Ie='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/iou.png" alt="IoU"/>',re,H,Se=`<li><strong>Pixel accuracy</strong>: Pixel accuracy is calculated as the ratio of the number of correctly classified pixels to the total number of pixels.
While being an intuitive metric, it can be misleading due to its sensitivity to class imbalance.</li>`,me,_,Pe='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/pixel-accuracy.png" alt="Pixel accuracy"/>',he,U,je=`<li><strong>Dice coefficient</strong>: It’s the ratio between the double of the intersection and the sum of the predicted mask and the ground truth.
The dice coefficient is simply the percentage of overlap between the prediction and the ground truth. It’s a good metric to use when
you need sensibility to small differences between the overlap.</li>`,pe,I,Je='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/dice-coefficient.png" alt="Dice coefficient"/>',ce,S,fe,P,We='<li><a href="https://arxiv.org/abs/2304.02643" rel="nofollow">Segment Anything Paper</a></li> <li><a href="https://huggingface.co/blog/fine-tune-segformer" rel="nofollow">Fine-tuning Segformer blog post</a></li> <li><a href="https://huggingface.co/blog/mask2former" rel="nofollow">Mask2former blog post</a></li> <li><a href="https://huggingface.co/docs/transformers/main/tasks/semantic_segmentation" rel="nofollow">Hugging Face’s documentation on segmentation tasks</a></li> <li>If you want to go deeper into the topic, we recommend you to check out Stanford’s <a href="https://www.youtube.com/watch?v=nDPWywWRIRo" rel="nofollow">lecture on segmentation</a>.</li>',ge,j,de,A,ue;return h=new Ve({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),p=new ve({props:{title:"Image Segmentation",local:"image-segmentation",headingTag:"h1"}}),w=new ve({props:{title:"Modern Approach: Vision Transformer-based Segmentation",local:"modern-approach-vision-transformer-based-segmentation",headingTag:"h3"}}),M=new Qe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBcGlwZSUyMCUzRCUyMHBpcGVsaW5lKCUyMm1hc2stZ2VuZXJhdGlvbiUyMiUyQyUyMG1vZGVsJTNEJTIyZmFjZWJvb2slMkZzYW0tdml0LWJhc2UlMjIlMkMlMjBkZXZpY2UlM0QwKSUwQSUwQXJhd19pbWFnZSUyMCUzRCUyMEltYWdlLm9wZW4oJTIycGF0aCUyRnRvJTJGaW1hZ2UlMjIpLmNvbnZlcnQoJTIyUkdCJTIyKSUwQSUwQW1hc2tzJTIwJTNEJTIwcGlwZShyYXdfaW1hZ2Up",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

pipe = pipeline(<span class="hljs-string">&quot;mask-generation&quot;</span>, model=<span class="hljs-string">&quot;facebook/sam-vit-base&quot;</span>, device=<span class="hljs-number">0</span>)

raw_image = Image.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;path/to/image&quot;</span>).convert(<span class="hljs-string">&quot;RGB&quot;</span>)

masks = pipe(raw_image)`,wrap:!1}}),x=new ve({props:{title:"How to Evaluate a Segmentation Model?",local:"how-to-evaluate-a-segmentation-model",headingTag:"h3"}}),S=new ve({props:{title:"Resources and Further Reading",local:"resources-and-further-reading",headingTag:"h2"}}),j=new Ye({props:{source:"https://github.com/huggingface/computer-vision-course/blob/main/chapters/en/unit6/basic-cv-tasks/segmentation.mdx"}}),{c(){m=o("meta"),E=a(),z=o("p"),F=a(),J(h.$$.fragment),B=a(),J(p.$$.fragment),V=a(),c=o("p"),c.innerHTML=ye,Q=a(),f=o("p"),f.innerHTML=be,Y=a(),g=o("p"),g.textContent=Te,D=a(),d=o("ul"),d.innerHTML=Me,N=a(),u=o("p"),u.innerHTML=$e,X=a(),v=o("p"),v.innerHTML=xe,K=a(),J(w.$$.fragment),O=a(),y=o("p"),y.textContent=ke,ee=a(),b=o("p"),b.innerHTML=Le,te=a(),T=o("p"),T.innerHTML=Ce,ne=a(),J(M.$$.fragment),ie=a(),$=o("p"),$.innerHTML=He,ae=a(),J(x.$$.fragment),se=a(),k=o("p"),k.textContent=_e,oe=a(),L=o("ul"),L.innerHTML=Ue,le=a(),C=o("p"),C.innerHTML=Ie,re=a(),H=o("ul"),H.innerHTML=Se,me=a(),_=o("p"),_.innerHTML=Pe,he=a(),U=o("ul"),U.innerHTML=je,pe=a(),I=o("p"),I.innerHTML=Je,ce=a(),J(S.$$.fragment),fe=a(),P=o("ul"),P.innerHTML=We,ge=a(),J(j.$$.fragment),de=a(),A=o("p"),this.h()},l(e){const t=Fe("svelte-u9bgzb",document.head);m=l(t,"META",{name:!0,content:!0}),t.forEach(n),E=s(e),z=l(e,"P",{}),Ze(z).forEach(n),F=s(e),W(h.$$.fragment,e),B=s(e),W(p.$$.fragment,e),V=s(e),c=l(e,"P",{"data-svelte-h":!0}),r(c)!=="svelte-exkdtj"&&(c.innerHTML=ye),Q=s(e),f=l(e,"P",{"data-svelte-h":!0}),r(f)!=="svelte-1bft94o"&&(f.innerHTML=be),Y=s(e),g=l(e,"P",{"data-svelte-h":!0}),r(g)!=="svelte-1w5fm6j"&&(g.textContent=Te),D=s(e),d=l(e,"UL",{"data-svelte-h":!0}),r(d)!=="svelte-n0x2m3"&&(d.innerHTML=Me),N=s(e),u=l(e,"P",{"data-svelte-h":!0}),r(u)!=="svelte-1beerg7"&&(u.innerHTML=$e),X=s(e),v=l(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-17sd42v"&&(v.innerHTML=xe),K=s(e),W(w.$$.fragment,e),O=s(e),y=l(e,"P",{"data-svelte-h":!0}),r(y)!=="svelte-19zwezj"&&(y.textContent=ke),ee=s(e),b=l(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-1334w5c"&&(b.innerHTML=Le),te=s(e),T=l(e,"P",{"data-svelte-h":!0}),r(T)!=="svelte-1bdv4ej"&&(T.innerHTML=Ce),ne=s(e),W(M.$$.fragment,e),ie=s(e),$=l(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-1kwqdq1"&&($.innerHTML=He),ae=s(e),W(x.$$.fragment,e),se=s(e),k=l(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-17c2odr"&&(k.textContent=_e),oe=s(e),L=l(e,"UL",{"data-svelte-h":!0}),r(L)!=="svelte-1ltwdtc"&&(L.innerHTML=Ue),le=s(e),C=l(e,"P",{"data-svelte-h":!0}),r(C)!=="svelte-wgpkoc"&&(C.innerHTML=Ie),re=s(e),H=l(e,"UL",{"data-svelte-h":!0}),r(H)!=="svelte-li3e7e"&&(H.innerHTML=Se),me=s(e),_=l(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-rdneb"&&(_.innerHTML=Pe),he=s(e),U=l(e,"UL",{"data-svelte-h":!0}),r(U)!=="svelte-1gypdy2"&&(U.innerHTML=je),pe=s(e),I=l(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-138bear"&&(I.innerHTML=Je),ce=s(e),W(S.$$.fragment,e),fe=s(e),P=l(e,"UL",{"data-svelte-h":!0}),r(P)!=="svelte-1g8ssyy"&&(P.innerHTML=We),ge=s(e),W(j.$$.fragment,e),de=s(e),A=l(e,"P",{}),Ze(A).forEach(n),this.h()},h(){Re(m,"name","hf:doc:metadata"),Re(m,"content",Ne)},m(e,t){Be(document.head,m),i(e,E,t),i(e,z,t),i(e,F,t),Z(h,e,t),i(e,B,t),Z(p,e,t),i(e,V,t),i(e,c,t),i(e,Q,t),i(e,f,t),i(e,Y,t),i(e,g,t),i(e,D,t),i(e,d,t),i(e,N,t),i(e,u,t),i(e,X,t),i(e,v,t),i(e,K,t),Z(w,e,t),i(e,O,t),i(e,y,t),i(e,ee,t),i(e,b,t),i(e,te,t),i(e,T,t),i(e,ne,t),Z(M,e,t),i(e,ie,t),i(e,$,t),i(e,ae,t),Z(x,e,t),i(e,se,t),i(e,k,t),i(e,oe,t),i(e,L,t),i(e,le,t),i(e,C,t),i(e,re,t),i(e,H,t),i(e,me,t),i(e,_,t),i(e,he,t),i(e,U,t),i(e,pe,t),i(e,I,t),i(e,ce,t),Z(S,e,t),i(e,fe,t),i(e,P,t),i(e,ge,t),Z(j,e,t),i(e,de,t),i(e,A,t),ue=!0},p:qe,i(e){ue||(R(h.$$.fragment,e),R(p.$$.fragment,e),R(w.$$.fragment,e),R(M.$$.fragment,e),R(x.$$.fragment,e),R(S.$$.fragment,e),R(j.$$.fragment,e),ue=!0)},o(e){G(h.$$.fragment,e),G(p.$$.fragment,e),G(w.$$.fragment,e),G(M.$$.fragment,e),G(x.$$.fragment,e),G(S.$$.fragment,e),G(j.$$.fragment,e),ue=!1},d(e){e&&(n(E),n(z),n(F),n(B),n(V),n(c),n(Q),n(f),n(Y),n(g),n(D),n(d),n(N),n(u),n(X),n(v),n(K),n(O),n(y),n(ee),n(b),n(te),n(T),n(ne),n(ie),n($),n(ae),n(se),n(k),n(oe),n(L),n(le),n(C),n(re),n(H),n(me),n(_),n(he),n(U),n(pe),n(I),n(ce),n(fe),n(P),n(ge),n(de),n(A)),n(m),q(h,e),q(p,e),q(w,e),q(M,e),q(x,e),q(S,e),q(j,e)}}}const Ne='{"title":"Image Segmentation","local":"image-segmentation","sections":[{"title":"Modern Approach: Vision Transformer-based Segmentation","local":"modern-approach-vision-transformer-based-segmentation","sections":[],"depth":3},{"title":"How to Evaluate a Segmentation Model?","local":"how-to-evaluate-a-segmentation-model","sections":[],"depth":3},{"title":"Resources and Further Reading","local":"resources-and-further-reading","sections":[],"depth":2}],"depth":1}';function Xe(we){return ze(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class it extends Ae{constructor(m){super(),Ee(this,m,Xe,De,Ge,{})}}export{it as component};
