import{s as xe,a as $e,n as Ze,o as Be}from"../chunks/scheduler.35aab934.js";import{S as Ve,i as We,e as i,s as a,c as p,h as Re,a as o,d as s,b as n,f as Ce,g as m,j as c,k as _,l as Ee,m as l,n as d,t as u,o as h,p as M}from"../chunks/index.de17b971.js";import{C as ke}from"../chunks/CopyLLMTxtMenu.48bc5cc2.js";import{C as Qe}from"../chunks/CodeBlock.c1f62de5.js";import{H,E as He}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.8fdece89.js";function _e(ue){let b,N,X,Y,j,z,f,F,y,O,T,he='Object detection, a critical task in computer vision, has seen significant advancements with models like the YOLO (<a href="https://arxiv.org/abs/1506.02640" rel="nofollow">original paper</a>, <a href="https://github.com/ultralytics/ultralytics" rel="nofollow">latest code version</a>). However, traditional models like YOLO have limitations in detecting objects outside their training datasets. To address this, the AI community has shifted towards developing models that can identify a wider range of objects, leading to the creation of models akin to CLIP but for object detection.',A,w,G,g,Me="OWL-ViT represents a leap forward in open-vocabulary object detection. It starts with a training stage similar to CLIP, focusing on a vision and language encoder using a contrastive loss. This foundational stage enables the model to learn a shared representation space for both visual and textual data.",q,J,P,U,be="The innovation in OWL-ViT comes during its fine-tuning stage for object detection. Here, instead of the token pooling and final projection layer used in CLIP, OWL-VIT employs a linear projection of each output token to obtain per-object image embeddings. These embeddings are then used for classification, while box coordinates are derived from token representations through a small MLP. This approach allows OWL-ViT to detect objects and their spatial locations within images, a significant advancement over traditional object detection models.",S,v,je="Below is a diagram of the Pre-training and Fine-tuning stages of OWL-ViT:",D,I,fe='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/owlvit_architecture.jpg" alt="OWL-ViT Pre-training and Fine-tuning"/>',K,C,ee,x,ye="After fine-tuning, OWL-ViT excels in open-vocabulary object detection. It can identify objects not explicitly present in the training dataset, thanks to the shared embedding space of the vision and text encoders. This capability makes it possible to use both images and textual queries for object detection, enhancing its versatility.",te,$,se,Z,Te="For practical applications, one typically uses text as the query and the image as context. The provided Python example demonstrates how to use the transformers library for running inference with OWL-ViT.",le,B,ae,V,we=`The image should look like this:
<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/OWL_ViT_example.jpg" alt="OWL-ViT Example"/>
This example shows a simple use case of OWL-ViT, where we specify objects likely in the dataset like cat and more abstract objects like cat tail which most likely aren’t in any object detection dataset. This shows the powerfulness of OWL-ViT in open-vocabulary object detection.`,ne,W,ge="Feel free to use the code to try more complex examples or this Gradio app:",ie,r,Je="",Ue,oe,R,re,E,ve="OWL-ViT’s approach to object detection represents a notable shift in how AI models understand and interact with the visual world. By integrating language understanding with visual perception, it pushes the boundaries of object detection, enabling more accurate and versatile models capable of identifying a broader range of objects. This evolution in model capabilities is crucial for applications requiring nuanced understanding of visual scenes, particularly in dynamic, real-world environments.",ce,k,Ie='For more in-depth information and technical details, refer to the <a href="https://arxiv.org/abs/2205.06230" rel="nofollow">OWL-VIT paper</a>. You can also find information about <a href="https://huggingface.co/docs/transformers/model_doc/owlv2" rel="nofollow">OWL-ViT 2 model</a> in the Hugging face documentation.',pe,Q,me,L,de;return j=new ke({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),f=new H({props:{title:"Multimodal Object Detection (OWL-ViT)",local:"multimodal-object-detection-owl-vit",headingTag:"h1"}}),y=new H({props:{title:"Introduction",local:"introduction",headingTag:"h3"}}),w=new H({props:{title:"OWL-ViT: Enhancements and Capabilities",local:"owl-vit-enhancements-and-capabilities",headingTag:"h3"}}),J=new H({props:{title:"Fine-tuning for Object Detection",local:"fine-tuning-for-object-detection",headingTag:"h4"}}),C=new H({props:{title:"Open-Vocabulary Detection",local:"open-vocabulary-detection",headingTag:"h4"}}),$=new H({props:{title:"Example Use Case",local:"example-use-case",headingTag:"h4"}}),B=new Qe({props:{code:"aW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMkMlMjBJbWFnZURyYXclMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBPd2xWaVRQcm9jZXNzb3IlMkMlMjBPd2xWaVRGb3JPYmplY3REZXRlY3Rpb24lMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBPd2xWaVRQcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRm93bHZpdC1iYXNlLXBhdGNoMzIlMjIpJTBBbW9kZWwlMjAlM0QlMjBPd2xWaVRGb3JPYmplY3REZXRlY3Rpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRm93bHZpdC1iYXNlLXBhdGNoMzIlMjIpJTBBJTBBdXJsJTIwJTNEJTIwJTIyaHR0cCUzQSUyRiUyRmltYWdlcy5jb2NvZGF0YXNldC5vcmclMkZ2YWwyMDE3JTJGMDAwMDAwMDM5NzY5LmpwZyUyMiUwQWltYWdlJTIwJTNEJTIwSW1hZ2Uub3BlbihyZXF1ZXN0cy5nZXQodXJsJTJDJTIwc3RyZWFtJTNEVHJ1ZSkucmF3KSUwQXRleHRzJTIwJTNEJTIwJTVCJTVCJTIyYSUyMHBob3RvJTIwb2YlMjBhJTIwY2F0JTIyJTJDJTIwJTIyYSUyMHBob3RvJTIwb2YlMjBhJTIwZG9nJTIyJTJDJTIwJTIycmVtb3RlJTIwY29udHJvbCUyMiUyQyUyMCUyMmNhdCUyMHRhaWwlMjIlNUQlNUQlMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IodGV4dCUzRHRleHRzJTJDJTIwaW1hZ2VzJTNEaW1hZ2UlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEElMEF0YXJnZXRfc2l6ZXMlMjAlM0QlMjB0b3JjaC5UZW5zb3IoJTVCaW1hZ2Uuc2l6ZSU1QiUzQSUzQS0xJTVEJTVEKSUwQXJlc3VsdHMlMjAlM0QlMjBwcm9jZXNzb3IucG9zdF9wcm9jZXNzX29iamVjdF9kZXRlY3Rpb24oJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUzRG91dHB1dHMlMkMlMjB0YXJnZXRfc2l6ZXMlM0R0YXJnZXRfc2l6ZXMlMkMlMjB0aHJlc2hvbGQlM0QwLjElMEEpJTBBaSUyMCUzRCUyMDAlMjAlMjAlMjMlMjBSZXRyaWV2ZSUyMHByZWRpY3Rpb25zJTIwZm9yJTIwdGhlJTIwZmlyc3QlMjBpbWFnZSUyMGZvciUyMHRoZSUyMGNvcnJlc3BvbmRpbmclMjB0ZXh0JTIwcXVlcmllcyUwQXRleHQlMjAlM0QlMjB0ZXh0cyU1QmklNUQlMEFib3hlcyUyQyUyMHNjb3JlcyUyQyUyMGxhYmVscyUyMCUzRCUyMHJlc3VsdHMlNUJpJTVEJTVCJTIyYm94ZXMlMjIlNUQlMkMlMjByZXN1bHRzJTVCaSU1RCU1QiUyMnNjb3JlcyUyMiU1RCUyQyUyMHJlc3VsdHMlNUJpJTVEJTVCJTIybGFiZWxzJTIyJTVEJTBBJTBBJTIzJTIwQ3JlYXRlJTIwYSUyMGRyYXclMjBvYmplY3QlMEFkcmF3JTIwJTNEJTIwSW1hZ2VEcmF3LkRyYXcoaW1hZ2UpJTBBJTBBJTIzJTIwRHJhdyUyMGVhY2glMjBib3VuZGluZyUyMGJveCUwQWZvciUyMGJveCUyQyUyMHNjb3JlJTJDJTIwbGFiZWwlMjBpbiUyMHppcChib3hlcyUyQyUyMHNjb3JlcyUyQyUyMGxhYmVscyklM0ElMEElMjAlMjAlMjAlMjBib3glMjAlM0QlMjAlNUJyb3VuZChpJTJDJTIwMiklMjBmb3IlMjBpJTIwaW4lMjBib3gudG9saXN0KCklNUQlMEElMjAlMjAlMjAlMjBwcmludCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmJTIyRGV0ZWN0ZWQlMjAlN0J0ZXh0JTVCbGFiZWwlNUQlN0QlMjB3aXRoJTIwY29uZmlkZW5jZSUyMCU3QnJvdW5kKHNjb3JlLml0ZW0oKSUyQyUyMDMpJTdEJTIwYXQlMjBsb2NhdGlvbiUyMCU3QmJveCU3RCUyMiUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjMlMjBEcmF3JTIwdGhlJTIwYm91bmRpbmclMjBib3glMjBvbiUyMHRoZSUyMGltYWdlJTBBJTIwJTIwJTIwJTIwZHJhdy5yZWN0YW5nbGUoYm94JTJDJTIwb3V0bGluZSUzRCUyMnJlZCUyMiklMEElMEElMjMlMjBEaXNwbGF5JTIwdGhlJTIwaW1hZ2UlMEFpbWFnZQ==",highlighted:`<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image, ImageDraw
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> OwlViTProcessor, OwlViTForObjectDetection

processor = OwlViTProcessor.from_pretrained(<span class="hljs-string">&quot;google/owlvit-base-patch32&quot;</span>)
model = OwlViTForObjectDetection.from_pretrained(<span class="hljs-string">&quot;google/owlvit-base-patch32&quot;</span>)

url = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000039769.jpg&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)
texts = [[<span class="hljs-string">&quot;a photo of a cat&quot;</span>, <span class="hljs-string">&quot;a photo of a dog&quot;</span>, <span class="hljs-string">&quot;remote control&quot;</span>, <span class="hljs-string">&quot;cat tail&quot;</span>]]
inputs = processor(text=texts, images=image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
outputs = model(**inputs)

target_sizes = torch.Tensor([image.size[::-<span class="hljs-number">1</span>]])
results = processor.post_process_object_detection(
    outputs=outputs, target_sizes=target_sizes, threshold=<span class="hljs-number">0.1</span>
)
i = <span class="hljs-number">0</span>  <span class="hljs-comment"># Retrieve predictions for the first image for the corresponding text queries</span>
text = texts[i]
boxes, scores, labels = results[i][<span class="hljs-string">&quot;boxes&quot;</span>], results[i][<span class="hljs-string">&quot;scores&quot;</span>], results[i][<span class="hljs-string">&quot;labels&quot;</span>]

<span class="hljs-comment"># Create a draw object</span>
draw = ImageDraw.Draw(image)

<span class="hljs-comment"># Draw each bounding box</span>
<span class="hljs-keyword">for</span> box, score, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(boxes, scores, labels):
    box = [<span class="hljs-built_in">round</span>(i, <span class="hljs-number">2</span>) <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> box.tolist()]
    <span class="hljs-built_in">print</span>(
        <span class="hljs-string">f&quot;Detected <span class="hljs-subst">{text[label]}</span> with confidence <span class="hljs-subst">{<span class="hljs-built_in">round</span>(score.item(), <span class="hljs-number">3</span>)}</span> at location <span class="hljs-subst">{box}</span>&quot;</span>
    )
    <span class="hljs-comment"># Draw the bounding box on the image</span>
    draw.rectangle(box, outline=<span class="hljs-string">&quot;red&quot;</span>)

<span class="hljs-comment"># Display the image</span>
image`,wrap:!1}}),R=new H({props:{title:"Conclusion",local:"conclusion",headingTag:"h3"}}),Q=new He({props:{source:"https://github.com/huggingface/computer-vision-course/blob/main/chapters/en/unit4/multimodal-models/clip-and-relatives/owl_vit.mdx"}}),{c(){b=i("meta"),N=a(),X=i("p"),Y=a(),p(j.$$.fragment),z=a(),p(f.$$.fragment),F=a(),p(y.$$.fragment),O=a(),T=i("p"),T.innerHTML=he,A=a(),p(w.$$.fragment),G=a(),g=i("p"),g.textContent=Me,q=a(),p(J.$$.fragment),P=a(),U=i("p"),U.textContent=be,S=a(),v=i("p"),v.textContent=je,D=a(),I=i("p"),I.innerHTML=fe,K=a(),p(C.$$.fragment),ee=a(),x=i("p"),x.textContent=ye,te=a(),p($.$$.fragment),se=a(),Z=i("p"),Z.textContent=Te,le=a(),p(B.$$.fragment),ae=a(),V=i("p"),V.innerHTML=we,ne=a(),W=i("p"),W.textContent=ge,ie=a(),r=i("iframe"),r.innerHTML=Je,oe=a(),p(R.$$.fragment),re=a(),E=i("p"),E.textContent=ve,ce=a(),k=i("p"),k.innerHTML=Ie,pe=a(),p(Q.$$.fragment),me=a(),L=i("p"),this.h()},l(e){const t=Re("svelte-u9bgzb",document.head);b=o(t,"META",{name:!0,content:!0}),t.forEach(s),N=n(e),X=o(e,"P",{}),Ce(X).forEach(s),Y=n(e),m(j.$$.fragment,e),z=n(e),m(f.$$.fragment,e),F=n(e),m(y.$$.fragment,e),O=n(e),T=o(e,"P",{"data-svelte-h":!0}),c(T)!=="svelte-rcp1v5"&&(T.innerHTML=he),A=n(e),m(w.$$.fragment,e),G=n(e),g=o(e,"P",{"data-svelte-h":!0}),c(g)!=="svelte-z4tskn"&&(g.textContent=Me),q=n(e),m(J.$$.fragment,e),P=n(e),U=o(e,"P",{"data-svelte-h":!0}),c(U)!=="svelte-v134fy"&&(U.textContent=be),S=n(e),v=o(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-kfdaj8"&&(v.textContent=je),D=n(e),I=o(e,"P",{"data-svelte-h":!0}),c(I)!=="svelte-yoirbd"&&(I.innerHTML=fe),K=n(e),m(C.$$.fragment,e),ee=n(e),x=o(e,"P",{"data-svelte-h":!0}),c(x)!=="svelte-tptnjo"&&(x.textContent=ye),te=n(e),m($.$$.fragment,e),se=n(e),Z=o(e,"P",{"data-svelte-h":!0}),c(Z)!=="svelte-z24mom"&&(Z.textContent=Te),le=n(e),m(B.$$.fragment,e),ae=n(e),V=o(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-85rdgm"&&(V.innerHTML=we),ne=n(e),W=o(e,"P",{"data-svelte-h":!0}),c(W)!=="svelte-1n8z8mw"&&(W.textContent=ge),ie=n(e),r=o(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0,"data-svelte-h":!0}),c(r)!=="svelte-iir95p"&&(r.innerHTML=Je),oe=n(e),m(R.$$.fragment,e),re=n(e),E=o(e,"P",{"data-svelte-h":!0}),c(E)!=="svelte-gx5huh"&&(E.textContent=ve),ce=n(e),k=o(e,"P",{"data-svelte-h":!0}),c(k)!=="svelte-1y0r6n7"&&(k.innerHTML=Ie),pe=n(e),m(Q.$$.fragment,e),me=n(e),L=o(e,"P",{}),Ce(L).forEach(s),this.h()},h(){_(b,"name","hf:doc:metadata"),_(b,"content",Xe),$e(r.src,Ue="https://johko-OWL-ViT.hf.space")||_(r,"src",Ue),_(r,"frameborder","0"),_(r,"width","850"),_(r,"height","450")},m(e,t){Ee(document.head,b),l(e,N,t),l(e,X,t),l(e,Y,t),d(j,e,t),l(e,z,t),d(f,e,t),l(e,F,t),d(y,e,t),l(e,O,t),l(e,T,t),l(e,A,t),d(w,e,t),l(e,G,t),l(e,g,t),l(e,q,t),d(J,e,t),l(e,P,t),l(e,U,t),l(e,S,t),l(e,v,t),l(e,D,t),l(e,I,t),l(e,K,t),d(C,e,t),l(e,ee,t),l(e,x,t),l(e,te,t),d($,e,t),l(e,se,t),l(e,Z,t),l(e,le,t),d(B,e,t),l(e,ae,t),l(e,V,t),l(e,ne,t),l(e,W,t),l(e,ie,t),l(e,r,t),l(e,oe,t),d(R,e,t),l(e,re,t),l(e,E,t),l(e,ce,t),l(e,k,t),l(e,pe,t),d(Q,e,t),l(e,me,t),l(e,L,t),de=!0},p:Ze,i(e){de||(u(j.$$.fragment,e),u(f.$$.fragment,e),u(y.$$.fragment,e),u(w.$$.fragment,e),u(J.$$.fragment,e),u(C.$$.fragment,e),u($.$$.fragment,e),u(B.$$.fragment,e),u(R.$$.fragment,e),u(Q.$$.fragment,e),de=!0)},o(e){h(j.$$.fragment,e),h(f.$$.fragment,e),h(y.$$.fragment,e),h(w.$$.fragment,e),h(J.$$.fragment,e),h(C.$$.fragment,e),h($.$$.fragment,e),h(B.$$.fragment,e),h(R.$$.fragment,e),h(Q.$$.fragment,e),de=!1},d(e){e&&(s(N),s(X),s(Y),s(z),s(F),s(O),s(T),s(A),s(G),s(g),s(q),s(P),s(U),s(S),s(v),s(D),s(I),s(K),s(ee),s(x),s(te),s(se),s(Z),s(le),s(ae),s(V),s(ne),s(W),s(ie),s(r),s(oe),s(re),s(E),s(ce),s(k),s(pe),s(me),s(L)),s(b),M(j,e),M(f,e),M(y,e),M(w,e),M(J,e),M(C,e),M($,e),M(B,e),M(R,e),M(Q,e)}}}const Xe='{"title":"Multimodal Object Detection (OWL-ViT)","local":"multimodal-object-detection-owl-vit","sections":[{"title":"Introduction","local":"introduction","sections":[],"depth":3},{"title":"OWL-ViT: Enhancements and Capabilities","local":"owl-vit-enhancements-and-capabilities","sections":[{"title":"Fine-tuning for Object Detection","local":"fine-tuning-for-object-detection","sections":[],"depth":4},{"title":"Open-Vocabulary Detection","local":"open-vocabulary-detection","sections":[],"depth":4},{"title":"Example Use Case","local":"example-use-case","sections":[],"depth":4}],"depth":3},{"title":"Conclusion","local":"conclusion","sections":[],"depth":3}],"depth":1}';function Le(ue){return Be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ae extends Ve{constructor(b){super(),We(this,b,Le,_e,xe,{})}}export{Ae as component};
