import{s as Ut,n as dt,o as jt}from"../chunks/scheduler.eb244325.js";import{S as ut,i as mt,e as M,s as n,c as y,h as It,a as p,d as e,b as a,f as he,g as i,j as o,k as xl,l as Ce,m as t,n as w,t as J,o as T,p as c}from"../chunks/index.661680a1.js";import{C as ht,H as U,E as Ct}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f2fe41ba.js";import{C as r}from"../chunks/CodeBlock.a220ed00.js";function bt(be){let d,zl,Yl,$l,b,ql,B,Dl,S,Be="If the model you’re looking to deploy isn’t supported by any of the high-performance inference engines (vLLM, SGLang, etc.), or you have <em>custom inference logic</em>, need <em>specific Python dependencies</em>, you can deploy a <strong>custom Docker container</strong> on <strong>Inference Endpoints</strong>.",Pl,f,Se="This requires more upfront work & understanding of running models in production but gives you full control over the hardware and the server.",Kl,Z,fe="We’ll walk you through a simple guide on how to:",Ol,g,Ze='<li>build a FastAPI server to run <a href="https://huggingface.co/HuggingFaceTB/SmolLM3-3B" rel="nofollow"><code>HuggingFaceTB/SmolLM3-3B</code></a></li> <li>containerize the server</li> <li>deploy the container on Inference Endpoints</li>',ls,k,ge="Let’s get to it! 😎",ss,L,es,E,ts,G,ke="Start by creating a new uv project by running:",ns,A,as,j,Le="<p>We’ll be using <code>uv</code> to build this project but using <code>pip</code> or <code>conda</code> works as well, just adjust the commands accordingly.</p>",Ms,V,Ee="The <code>main.py</code> file will:",ps,W,Ge="<li>load the model from <code>/repository</code></li> <li>start a FastAPI app</li> <li>expose a <code>/health</code> and a <code>/generate</code> route</li>",os,u,Ae="<p>Inference Endpoints has a way to download model artifacts super fast, so ideally our code doesn’t download anything related to the model. The model you select when creating the endpoint will be mounted at <code>/repository</code>. <strong>So always load your model from <code>/repository</code></strong>, not directly from the Hugging Face Hub.</p>",ys,v,is,R,Ve="Before getting to the code, let’s install the necessary dependencies:",ws,X,Js,N,We="Now let’s build the code step by step. We’ll start by adding all imports and declare a few global variables. The <code>DEVICE</code> and <code>DTYPE</code> global variables are dynamically set according to the underlying GPU/CPU hardware availability.",Ts,Q,cs,_,rs,x,Us,F,ve="We will follow a few best practices:",ds,Y,Re=`<li><p><strong>ModelManager</strong>
Avoid keeping raw global model/tokenizer objects without lifecycle control. A small <code>ModelManager</code> class lets us:</p> <ul><li>eagerly <strong>load</strong> the model onto the accelerator, and</li> <li>safely <strong>unload</strong> it and free memory when the server shuts down.</li></ul> <p>The benefit we get here is that we can control the server’s behaviour based on the state of the model and tokenizer.
We want the server to start → load the model &amp; tokenizer → then signal that the server is ready for requests.</p> <p>For convenience, we also create a small <code>ModelNotLoadedError</code> class, to be able to communicate more clearly when the model &amp; tokenizer aren’t loaded.</p></li>`,js,H,us,z,ms,m,Xe=`<li><strong>FastAPI lifespan</strong>
Using FastAPI’s <code>lifespan</code> we can tie the model manager’s functionality to the server by:
<ul><li>loading the model on app startup using <code>model_manager.load()</code></li> <li>unloading the model on app shutdown using <code>model_manager.unload()</code>
This keeps your server’s memory usage clean and predictable.</li></ul></li>`,Is,$,hs,q,Cs,D,Ne="Now that we have the lifecycle in place, we can start building the core logic of the server itself. We’ll start by defining the request and response types, so that we know exactly what type of data we can pass in to the server and what to expect in response.",bs,P,Qe="The default value for <code>max_new_tokens</code> is <code>128</code> and can be increased to a maximum of <code>512</code>. This is a practical way of capping the maximum memory a request can take.",Bs,K,Ss,O,_e='Feel free to extend the parameters to include <code>temperature</code>, <code>top_p</code> and other <a href="https://huggingface.co/docs/transformers/v5.0.0rc0/en/model_doc/smollm3#transformers.SmolLM3ForCausalLM" rel="nofollow">configurations supported</a> by the model.',fs,ll,Zs,sl,xe="Moving on to creating the routes for the server - let’s start with the <code>/health</code> route. Here, we’re finally using the model manager to know if the model and tokenizer are ready to go. If the model manager returns a <code>ModelNotLoadedError</code>, we also return an error with the status code of <code>503</code>.",gs,el,Fe="On Inference Endpoints (and most other platforms), a <em>readiness probe</em> will ping an endpoint every second on its <code>/health</code> route, to check that everything is okay. Using this pattern we can clearly signal that the server isn’t ready before the models and tokenizer are fully initialized.",ks,tl,Ls,nl,Ye="And finally the most interesting section: the <code>/generate</code> route. This is the route that we want to call to actually use the model for text generation.",Es,al,He="<li>It starts with a similar guard as the <code>/health</code> route: we check that the model and tokenizer are loaded, and if not, return a <code>503</code> error.</li> <li>We assume that the model supports <code>apply_chat_template</code>, but fall back to passing the prompt directly without chat templating.</li> <li>We encode the text to tokens and call <code>model.generate()</code>.</li> <li>Lastly, we gather the outputs, decode the tokens to text, and return the response.</li>",Gs,Ml,As,pl,Vs,ol,ze="If you want to run the server locally you would need to replace:",Ws,yl,vs,il,$e="Since locally we actually do want to download the model from the Hugging Face Hub. But don’t forget to change this back!",Rs,wl,qe="and then run the following:",Xs,Jl,Ns,Tl,De="Go to <code>http://127.0.0.1:8000/docs</code> to see the automatic documentation that FastAPI provides.",Qs,cl,Pe="Well done 🙌",_s,rl,xs,I,Fl,Ke="If you want to copy & paste the full code you'll find it here:",Ie,Ul,Fs,dl,Ys,jl,Oe="Now let’s create a <code>Dockerfile</code> to package our server into a container.",Hs,h,lt="<p>Model weights shouldn’t be baked into the image: Inference Endpoints will mount the selected model at <code>/repository</code>, so the image only needs your <strong>code</strong> and <strong>dependencies</strong>.</p>",zs,ul,st="We’ll also avoid running as <code>root</code> inside the container by creating a non-root user and granting it access to <code>/app</code>.",$s,ml,et="First if your uv project doesn’t have a lockfile, which is common if you just created it, we can manually tell uv to make one for us by running:",qs,Il,Ds,hl,tt="Our Dockerfile will be very standard:",Ps,Cl,nt="<li>We use the base PyTorch image with CUDA and cuDNN</li> <li>We copy the uv binary</li> <li>Make sure that we’re not running things as a privileged user</li> <li>Install the dependencies with uv</li> <li>Make sure that we expose the correct port</li> <li>Run the server</li>",Ks,bl,Os,Bl,le,Sl,at="Once your <code>Dockerfile</code> and <code>main.py</code> are ready, build the container and push it to a registry that Inference Endpoints can access (Docker Hub, Amazon ECR, Azure ACR, or Google GCR).",se,fl,ee,C,Mt="<p>Why <code>--platform linux/amd64</code>? If you’re building this image on a Mac, it will automatically be built for an <code>arm64</code> machine, which is not a supported architecture for Inference Endpoints. That is why we need this flag to specify that we’re targeting the <code>x86</code> architecture. If you’re on an <code>x86</code> machine already, you can ignore this flag.</p>",te,Zl,ne,gl,pt="Now switch to the Inference Endpoints UI and deploy your custom container.",ae,kl,ot=`<li><p>Open the <a href="https://endpoints.huggingface.co/" rel="nofollow">Inference Endpoints dashboard</a> and click <strong>”+ New”</strong>.
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/endpoint-new.png" alt="endpoint-new.png"/></p></li> <li><p>Select <code>HuggingFaceTB/SmolLM3-3B</code> as the model repository (this will be mounted at <code>/repository</code> inside the container).
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/choose-smollm.png" alt="choose-smollm.png"/></p></li> <li><p>Click <strong>“Configure”</strong> to proceed with the deployment setup.
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/configure.png" alt="configure.png"/></p></li> <li><p>This is the configuration page where you’ll define compute, networking, and container settings.
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/home.png" alt="home.png"/></p></li> <li><p>Choose the hardware. Let’s go with the suggested L4.
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/authenticated.png" alt="authenticated.png"/></p></li> <li><p>Under <strong>Custom Container</strong>, enter:</p> <ul><li>your image URL (e.g., <code>your-username/smollm-endpoint:v0.1.0</code>)</li> <li>the port exposed by your container (in our case <code>8000</code>)
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/custom.png" alt="custom.png"/></li></ul></li> <li><p>Click <strong>“Create Endpoint”</strong>. The platform will:</p> <ul><li>pull your container image</li> <li>mount the model at <code>/repository</code></li> <li>start your FastAPI server
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/initializing.png" alt="initializing.png"/></li></ul></li> <li><p>After a short initialization period, the status will change to <strong>Running</strong>. Your custom container is now serving requests.
<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/running.png" alt="running.png"/></p></li>`,Me,Ll,yt="Once deployed, your endpoint will be available at a URL like:",pe,El,oe,Gl,it="Below is a minimal Python client you can use to test it:",ye,Al,ie,Vl,wt="If you open the <strong>Logs</strong> tab of your endpoint, you should see the incoming POST request and the model’s response.",we,Wl,Jt='<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/custom_container/post.png" alt="post.png"/>',Je,vl,Te,Rl,ce,Xl,Tt="Congratulations for making it until the end. 🎉",re,Nl,ct="A good idea to extend this demo would be to test it out with a completely different model, say an audio model or image generation one.",Ue,Ql,rt="Happy hacking 🙌",de,_l,je,Hl,ue;return b=new ht({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),B=new U({props:{title:"Deploy with your own container",local:"deploy-with-your-own-container",headingTag:"h1"}}),L=new U({props:{title:"1. Create the inference server",local:"1-create-the-inference-server",headingTag:"h2"}}),E=new U({props:{title:"1.1 Initialize the uv project",local:"11-initialize-the-uv-project",headingTag:"h3"}}),A=new r({props:{code:"dXYlMjBpbml0JTIwaW5mZXJlbmNlLXNlcnZlcg==",highlighted:"uv init inference-server",wrap:!1}}),v=new U({props:{title:"1.2 Install the Python dependencies",local:"12-install-the-python-dependencies",headingTag:"h3"}}),X=new r({props:{code:"dXYlMjBhZGQlMjB0cmFuc2Zvcm1lcnMlMjB0b3JjaCUyMCUyMmZhc3RhcGklNUJzdGFuZGFyZCU1RCUyMg==",highlighted:'uv add transformers torch <span class="hljs-string">&quot;fastapi[standard]&quot;</span>',wrap:!1}}),Q=new U({props:{title:"1.3 Add configurations",local:"13-add-configurations",headingTag:"h3"}}),_=new r({props:{code:"aW1wb3J0JTIwbG9nZ2luZyUwQWltcG9ydCUyMHRpbWUlMEFmcm9tJTIwY29udGV4dGxpYiUyMGltcG9ydCUyMGFzeW5jY29udGV4dG1hbmFnZXIlMEFmcm9tJTIwdHlwaW5nJTIwaW1wb3J0JTIwT3B0aW9uYWwlMEElMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjBmYXN0YXBpJTIwaW1wb3J0JTIwRmFzdEFQSSUyQyUyMEhUVFBFeGNlcHRpb24lMEFmcm9tJTIwcHlkYW50aWMlMjBpbXBvcnQlMjBCYXNlTW9kZWwlMkMlMjBGaWVsZCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEElMEElMjMlMjAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMEElMjMlMjBDb25maWclMjAlMkIlMjBMb2dnaW5nJTBBJTIzJTIwLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTBBTU9ERUxfSUQlMjAlM0QlMjAlMjIlMkZyZXBvc2l0b3J5JTIyJTBBREVWSUNFJTIwJTNEJTIwdG9yY2guZGV2aWNlKCUyMmN1ZGElMjIlMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwJTIyY3B1JTIyKSUwQURUWVBFJTIwJTNEJTIwdG9yY2guYmZsb2F0MTYlMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwdG9yY2guZmxvYXQzMiUwQU1BWF9ORVdfVE9LRU5TJTIwJTNEJTIwNTEyJTBBJTBBbG9nZ2luZy5iYXNpY0NvbmZpZyhsZXZlbCUzRGxvZ2dpbmcuSU5GTyklMEFsb2dnZXIlMjAlM0QlMjBsb2dnaW5nLmdldExvZ2dlcihfX25hbWVfXyk=",highlighted:`<span class="hljs-keyword">import</span> logging
<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">from</span> contextlib <span class="hljs-keyword">import</span> asynccontextmanager
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">Optional</span>

<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> fastapi <span class="hljs-keyword">import</span> FastAPI, HTTPException
<span class="hljs-keyword">from</span> pydantic <span class="hljs-keyword">import</span> BaseModel, Field
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer


<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Config + Logging</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
MODEL_ID = <span class="hljs-string">&quot;/repository&quot;</span>
DEVICE = torch.device(<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>)
DTYPE = torch.bfloat16 <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.float32
MAX_NEW_TOKENS = <span class="hljs-number">512</span>

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)`,wrap:!1}}),x=new U({props:{title:"1.4 Implement the ModelManager",local:"14-implement-the-modelmanager",headingTag:"h3"}}),H=new r({props:{code:" ",highlighted:`<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Model Manager</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">ModelNotLoadedError</span>(<span class="hljs-title class_ inherited__">RuntimeError</span>):
    <span class="hljs-string">&quot;&quot;&quot;Raised when attempting to use the model before it is loaded.&quot;&quot;&quot;</span>


<span class="hljs-keyword">class</span> <span class="hljs-title class_">ModelManager</span>:
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, model_id: <span class="hljs-built_in">str</span>, device: <span class="hljs-built_in">str</span>, dtype: torch.dtype</span>):
        self.model_id = model_id
        self.device = device
        self.dtype = dtype

        self.model: <span class="hljs-type">Optional</span>[AutoModelForCausalLM] = <span class="hljs-literal">None</span>
        self.tokenizer: <span class="hljs-type">Optional</span>[AutoTokenizer] = <span class="hljs-literal">None</span>

    <span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">load</span>(<span class="hljs-params">self</span>):
        <span class="hljs-string">&quot;&quot;&quot;Load model + tokenizer if not already loaded.&quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.model <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> self.tokenizer <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
            <span class="hljs-keyword">return</span>

        start = time.perf_counter()
        logger.info(<span class="hljs-string">f&quot;Loading tokenizer and model for <span class="hljs-subst">{self.model_id}</span>&quot;</span>)

        self.tokenizer = AutoTokenizer.from_pretrained(
            self.model_id,
        )
        self.model = (
            AutoModelForCausalLM.from_pretrained(
                self.model_id,
                dtype=self.dtype,
            )
            .to(self.device)
            .<span class="hljs-built_in">eval</span>()
        )
        duration_ms = (time.perf_counter() - start) * <span class="hljs-number">1000</span>
        logger.info(<span class="hljs-string">f&quot;Finished loading <span class="hljs-subst">{self.model_id}</span> in <span class="hljs-subst">{duration_ms:<span class="hljs-number">.2</span>f}</span> ms&quot;</span>)

    <span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">unload</span>(<span class="hljs-params">self</span>):
        <span class="hljs-string">&quot;&quot;&quot;Free model + tokenizer and clear CUDA cache.&quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.model <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
            self.model.to(<span class="hljs-string">&quot;cpu&quot;</span>)
            <span class="hljs-keyword">del</span> self.model
            self.model = <span class="hljs-literal">None</span>

        <span class="hljs-keyword">if</span> self.tokenizer <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
            <span class="hljs-keyword">del</span> self.tokenizer
            self.tokenizer = <span class="hljs-literal">None</span>

        <span class="hljs-keyword">if</span> torch.cuda.is_available():
            torch.cuda.empty_cache()

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get</span>(<span class="hljs-params">self</span>):
        <span class="hljs-string">&quot;&quot;&quot;Return the loaded model + tokenizer or raise if not ready.&quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.model <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> self.tokenizer <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
            <span class="hljs-keyword">raise</span> ModelNotLoadedError(<span class="hljs-string">&quot;Model not loaded&quot;</span>)
        <span class="hljs-keyword">return</span> self.model, self.tokenizer


model_manager = ModelManager(MODEL_ID, DEVICE, DTYPE)`,wrap:!1}}),z=new U({props:{title:"1.5 Use FastAPI lifespan for startup and shutdown",local:"15-use-fastapi-lifespan-for-startup-and-shutdown",headingTag:"h3"}}),$=new r({props:{code:"JTIzJTIwLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTBBJTIzJTIwTGlmZXNwYW4lMjAoc3RhcnR1cCUyMCUyQiUyMHNodXRkb3duKSUwQSUyMyUyMC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSUwQSU0MGFzeW5jY29udGV4dG1hbmFnZXIlMEFhc3luYyUyMGRlZiUyMGxpZmVzcGFuKGFwcCUzQSUyMEZhc3RBUEkpJTNBJTBBJTIwJTIwJTIwJTIwYXdhaXQlMjBtb2RlbF9tYW5hZ2VyLmxvYWQoKSUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHlpZWxkJTBBJTIwJTIwJTIwJTIwZmluYWxseSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGF3YWl0JTIwbW9kZWxfbWFuYWdlci51bmxvYWQoKSUwQSUwQSUwQWFwcCUyMCUzRCUyMEZhc3RBUEkobGlmZXNwYW4lM0RsaWZlc3Bhbik=",highlighted:`<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Lifespan (startup + shutdown)</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-meta">@asynccontextmanager</span>
<span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">lifespan</span>(<span class="hljs-params">app: FastAPI</span>):
    <span class="hljs-keyword">await</span> model_manager.load()
    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">yield</span>
    <span class="hljs-keyword">finally</span>:
        <span class="hljs-keyword">await</span> model_manager.unload()


app = FastAPI(lifespan=lifespan)`,wrap:!1}}),q=new U({props:{title:"1.6 Define the request and response schemas",local:"16-define-the-request-and-response-schemas",headingTag:"h3"}}),K=new r({props:{code:"JTIzJTIwLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTBBJTIzJTIwU2NoZW1hcyUwQSUyMyUyMC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSUwQWNsYXNzJTIwR2VuZXJhdGVSZXF1ZXN0KEJhc2VNb2RlbCklM0ElMEElMjAlMjAlMjAlMjBwcm9tcHQlM0ElMjBzdHIlMjAlM0QlMjBGaWVsZCguLi4lMkMlMjBtaW5fbGVuZ3RoJTNEMSUyQyUyMGRlc2NyaXB0aW9uJTNEJTIyUGxhaW4tdGV4dCUyMHByb21wdCUyMiklMEElMjAlMjAlMjAlMjBtYXhfbmV3X3Rva2VucyUzQSUyMGludCUyMCUzRCUyMEZpZWxkKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMDEyOCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGdlJTNEMSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxlJTNETUFYX05FV19UT0tFTlMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkZXNjcmlwdGlvbiUzRCUyMlVwcGVyJTIwYm91bmQlMjBvbiUyMGdlbmVyYXRlZCUyMHRva2VucyUyMiUyQyUwQSUyMCUyMCUyMCUyMCklMEElMEElMEFjbGFzcyUyMEdlbmVyYXRlUmVzcG9uc2UoQmFzZU1vZGVsKSUzQSUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlJTNBJTIwc3RyJTBBJTIwJTIwJTIwJTIwaW5wdXRfdG9rZW5fY291bnQlM0ElMjBpbnQlMEElMjAlMjAlMjAlMjBvdXRwdXRfdG9rZW5fY291bnQlM0ElMjBpbnQ=",highlighted:`<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Schemas</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">GenerateRequest</span>(<span class="hljs-title class_ inherited__">BaseModel</span>):
    prompt: <span class="hljs-built_in">str</span> = Field(..., min_length=<span class="hljs-number">1</span>, description=<span class="hljs-string">&quot;Plain-text prompt&quot;</span>)
    max_new_tokens: <span class="hljs-built_in">int</span> = Field(
        <span class="hljs-number">128</span>,
        ge=<span class="hljs-number">1</span>,
        le=MAX_NEW_TOKENS,
        description=<span class="hljs-string">&quot;Upper bound on generated tokens&quot;</span>,
    )


<span class="hljs-keyword">class</span> <span class="hljs-title class_">GenerateResponse</span>(<span class="hljs-title class_ inherited__">BaseModel</span>):
    response: <span class="hljs-built_in">str</span>
    input_token_count: <span class="hljs-built_in">int</span>
    output_token_count: <span class="hljs-built_in">int</span>`,wrap:!1}}),ll=new U({props:{title:"1.7 Implement the server routes",local:"17-implement-the-server-routes",headingTag:"h3"}}),tl=new r({props:{code:"JTIzJTIwLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTBBJTIzJTIwUm91dGVzJTBBJTIzJTIwLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tJTBBJTQwYXBwLmdldCglMjIlMkZoZWFsdGglMjIpJTBBZGVmJTIwaGVhbHRoKCklM0ElMEElMjAlMjAlMjAlMjB0cnklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9tYW5hZ2VyLmdldCgpJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwTW9kZWxOb3RMb2FkZWRFcnJvciUyMGFzJTIwZXhjJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmFpc2UlMjBIVFRQRXhjZXB0aW9uKHN0YXR1c19jb2RlJTNENTAzJTJDJTIwZGV0YWlsJTNEc3RyKGV4YykpJTIwZnJvbSUyMGV4YyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUyMm1lc3NhZ2UlMjIlM0ElMjAlMjJBUEklMjBpcyUyMHJ1bm5pbmcuJTIyJTdE",highlighted:`<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Routes</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-meta">@app.get(<span class="hljs-params"><span class="hljs-string">&quot;/health&quot;</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">health</span>():
    <span class="hljs-keyword">try</span>:
        model_manager.get()
    <span class="hljs-keyword">except</span> ModelNotLoadedError <span class="hljs-keyword">as</span> exc:
        <span class="hljs-keyword">raise</span> HTTPException(status_code=<span class="hljs-number">503</span>, detail=<span class="hljs-built_in">str</span>(exc)) <span class="hljs-keyword">from</span> exc
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;message&quot;</span>: <span class="hljs-string">&quot;API is running.&quot;</span>}`,wrap:!1}}),Ml=new r({props:{code:"JTQwYXBwLnBvc3QoJTIyJTJGZ2VuZXJhdGUlMjIlMkMlMjByZXNwb25zZV9tb2RlbCUzREdlbmVyYXRlUmVzcG9uc2UpJTBBZGVmJTIwZ2VuZXJhdGUocmVxdWVzdCUzQSUyMEdlbmVyYXRlUmVxdWVzdCklMjAtJTNFJTIwR2VuZXJhdGVSZXNwb25zZSUzQSUwQSUyMCUyMCUyMCUyMHN0YXJ0X3RpbWUlMjAlM0QlMjB0aW1lLnBlcmZfY291bnRlcigpJTBBJTIwJTIwJTIwJTIwdHJ5JTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjB0b2tlbml6ZXIlMjAlM0QlMjBtb2RlbF9tYW5hZ2VyLmdldCgpJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwTW9kZWxOb3RMb2FkZWRFcnJvciUyMGFzJTIwZXhjJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmFpc2UlMjBIVFRQRXhjZXB0aW9uKHN0YXR1c19jb2RlJTNENTAzJTJDJTIwZGV0YWlsJTNEc3RyKGV4YykpJTIwZnJvbSUyMGV4YyUwQSUwQSUyMCUyMCUyMCUyMGlmJTIwZ2V0YXR0cih0b2tlbml6ZXIlMkMlMjAlMjJjaGF0X3RlbXBsYXRlJTIyJTJDJTIwTm9uZSklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtZXNzYWdlcyUyMCUzRCUyMCU1QiU3QiUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMHJlcXVlc3QucHJvbXB0JTdEJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5wdXRfdGV4dCUyMCUzRCUyMHRva2VuaXplci5hcHBseV9jaGF0X3RlbXBsYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1lc3NhZ2VzJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdG9rZW5pemUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFkZF9nZW5lcmF0aW9uX3Byb21wdCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X3RleHQlMjAlM0QlMjByZXF1ZXN0LnByb21wdCUwQSUwQSUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMHRva2VuaXplcihpbnB1dF90ZXh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oREVWSUNFKSUwQSUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5pbmZlcmVuY2VfbW9kZSgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwbWF4X25ld190b2tlbnMlM0RyZXF1ZXN0Lm1heF9uZXdfdG9rZW5zKSUwQSUyMCUyMCUyMCUyMGV4Y2VwdCUyMFJ1bnRpbWVFcnJvciUyMGFzJTIwZXhjJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9nZ2VyLmV4Y2VwdGlvbiglMjJHZW5lcmF0aW9uJTIwZmFpbGVkJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJhaXNlJTIwSFRUUEV4Y2VwdGlvbiglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdGF0dXNfY29kZSUzRDUwMCUyQyUyMGRldGFpbCUzRGYlMjJHZW5lcmF0aW9uJTIwZmFpbGVkJTNBJTIwJTdCZXhjJTdEJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyMGZyb20lMjBleGMlMEElMEElMjAlMjAlMjAlMjBpbnB1dF90b2tlbl9jb3VudCUyMCUzRCUyMGlucHV0cy5pbnB1dF9pZHMuc2hhcGUlNUIxJTVEJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG91dHB1dHMlNUIwJTVEJTVCaW5wdXRfdG9rZW5fY291bnQlM0ElNUQlMEElMjAlMjAlMjAlMjBnZW5lcmF0ZWRfdGV4dCUyMCUzRCUyMHRva2VuaXplci5kZWNvZGUoZ2VuZXJhdGVkX2lkcyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMG91dHB1dF90b2tlbl9jb3VudCUyMCUzRCUyMGdlbmVyYXRlZF9pZHMuc2hhcGUlNUIwJTVEJTBBJTIwJTIwJTIwJTIwZHVyYXRpb25fbXMlMjAlM0QlMjAodGltZS5wZXJmX2NvdW50ZXIoKSUyMC0lMjBzdGFydF90aW1lKSUyMColMjAxMDAwJTBBJTBBJTIwJTIwJTIwJTIwbG9nZ2VyLmluZm8oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZiUyMmdlbmVyYXRlJTIwcHJvbXB0X3Rva2VucyUzRCU3QmlucHV0X3Rva2VuX2NvdW50JTdEJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZiUyMm5ld190b2tlbnMlM0QlN0JvdXRwdXRfdG9rZW5fY291bnQlN0QlMjBtYXhfbmV3X3Rva2VucyUzRCU3QnJlcXVlc3QubWF4X25ld190b2tlbnMlN0QlMjAlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmJTIyZHVyYXRpb25fbXMlM0QlN0JkdXJhdGlvbl9tcyUzQS4yZiU3RCUyMiUwQSUyMCUyMCUyMCUyMCklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBHZW5lcmF0ZVJlc3BvbnNlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3BvbnNlJTNEZ2VuZXJhdGVkX3RleHQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dF90b2tlbl9jb3VudCUzRGlucHV0X3Rva2VuX2NvdW50JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0X3Rva2VuX2NvdW50JTNEb3V0cHV0X3Rva2VuX2NvdW50JTJDJTBBJTIwJTIwJTIwJTIwKQ==",highlighted:`<span class="hljs-meta">@app.post(<span class="hljs-params"><span class="hljs-string">&quot;/generate&quot;</span>, response_model=GenerateResponse</span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate</span>(<span class="hljs-params">request: GenerateRequest</span>) -&gt; GenerateResponse:
    start_time = time.perf_counter()
    <span class="hljs-keyword">try</span>:
        model, tokenizer = model_manager.get()
    <span class="hljs-keyword">except</span> ModelNotLoadedError <span class="hljs-keyword">as</span> exc:
        <span class="hljs-keyword">raise</span> HTTPException(status_code=<span class="hljs-number">503</span>, detail=<span class="hljs-built_in">str</span>(exc)) <span class="hljs-keyword">from</span> exc

    <span class="hljs-keyword">if</span> <span class="hljs-built_in">getattr</span>(tokenizer, <span class="hljs-string">&quot;chat_template&quot;</span>, <span class="hljs-literal">None</span>):
        messages = [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: request.prompt}]
        input_text = tokenizer.apply_chat_template(
            messages,
            tokenize=<span class="hljs-literal">False</span>,
            add_generation_prompt=<span class="hljs-literal">True</span>,
        )
    <span class="hljs-keyword">else</span>:
        input_text = request.prompt

    inputs = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(DEVICE)

    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">with</span> torch.inference_mode():
            outputs = model.generate(**inputs, max_new_tokens=request.max_new_tokens)
    <span class="hljs-keyword">except</span> RuntimeError <span class="hljs-keyword">as</span> exc:
        logger.exception(<span class="hljs-string">&quot;Generation failed&quot;</span>)
        <span class="hljs-keyword">raise</span> HTTPException(
            status_code=<span class="hljs-number">500</span>, detail=<span class="hljs-string">f&quot;Generation failed: <span class="hljs-subst">{exc}</span>&quot;</span>
        ) <span class="hljs-keyword">from</span> exc

    input_token_count = inputs.input_ids.shape[<span class="hljs-number">1</span>]
    generated_ids = outputs[<span class="hljs-number">0</span>][input_token_count:]
    generated_text = tokenizer.decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)
    output_token_count = generated_ids.shape[<span class="hljs-number">0</span>]
    duration_ms = (time.perf_counter() - start_time) * <span class="hljs-number">1000</span>

    logger.info(
        <span class="hljs-string">f&quot;generate prompt_tokens=<span class="hljs-subst">{input_token_count}</span> &quot;</span>
        <span class="hljs-string">f&quot;new_tokens=<span class="hljs-subst">{output_token_count}</span> max_new_tokens=<span class="hljs-subst">{request.max_new_tokens}</span> &quot;</span>
        <span class="hljs-string">f&quot;duration_ms=<span class="hljs-subst">{duration_ms:<span class="hljs-number">.2</span>f}</span>&quot;</span>
    )

    <span class="hljs-keyword">return</span> GenerateResponse(
        response=generated_text,
        input_token_count=input_token_count,
        output_token_count=output_token_count,
    )`,wrap:!1}}),pl=new U({props:{title:"1.8 Run the server locally",local:"18-run-the-server-locally",headingTag:"h3"}}),yl=new r({props:{code:"LSUyME1PREVMX0lEJTIwJTNEJTIwJTIyJTJGcmVwb3NpdG9yeSUyMiUwQSUyQiUyME1PREVMX0lEJTIwJTNEJTIwJTIySHVnZ2luZ0ZhY2VUQiUyRlNtb2xMTTMtM0IlMjI=",highlighted:`<span class="hljs-deletion">- MODEL_ID = &quot;/repository&quot;</span>
<span class="hljs-addition">+ MODEL_ID = &quot;HuggingFaceTB/SmolLM3-3B&quot;</span>`,wrap:!1}}),Jl=new r({props:{code:"dXYlMjBydW4lMjB1dmljb3JuJTIwbWFpbiUzQWFwcA==",highlighted:"uv run uvicorn main:app",wrap:!1}}),rl=new U({props:{title:"1.9 Full server code listing",local:"19-full-server-code-listing",headingTag:"h3"}}),Ul=new r({props:{code:" ",highlighted:`<span class="hljs-keyword">import</span> logging
<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">from</span> contextlib <span class="hljs-keyword">import</span> asynccontextmanager
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">Optional</span>

<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> fastapi <span class="hljs-keyword">import</span> FastAPI, HTTPException
<span class="hljs-keyword">from</span> pydantic <span class="hljs-keyword">import</span> BaseModel, Field
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer


<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Config + Logging</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
MODEL_ID = <span class="hljs-string">&quot;/repository&quot;</span>
DEVICE = torch.device(<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>)
DTYPE = torch.bfloat16 <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.float32
MAX_NEW_TOKENS = <span class="hljs-number">512</span>

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Model Manager</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">ModelNotLoadedError</span>(<span class="hljs-title class_ inherited__">RuntimeError</span>):
    <span class="hljs-string">&quot;&quot;&quot;Raised when attempting to use the model before it is loaded.&quot;&quot;&quot;</span>


<span class="hljs-keyword">class</span> <span class="hljs-title class_">ModelManager</span>:
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, model_id: <span class="hljs-built_in">str</span>, device: <span class="hljs-built_in">str</span>, dtype: torch.dtype</span>):
        self.model_id = model_id
        self.device = device
        self.dtype = dtype

        self.model: <span class="hljs-type">Optional</span>[AutoModelForCausalLM] = <span class="hljs-literal">None</span>
        self.tokenizer: <span class="hljs-type">Optional</span>[AutoTokenizer] = <span class="hljs-literal">None</span>

    <span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">load</span>(<span class="hljs-params">self</span>):
        <span class="hljs-string">&quot;&quot;&quot;Load model + tokenizer if not already loaded.&quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.model <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> self.tokenizer <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
            <span class="hljs-keyword">return</span>

        start = time.perf_counter()
        logger.info(<span class="hljs-string">f&quot;Loading tokenizer and model for <span class="hljs-subst">{self.model_id}</span>&quot;</span>)

        self.tokenizer = AutoTokenizer.from_pretrained(
            self.model_id,
        )
        self.model = (
            AutoModelForCausalLM.from_pretrained(
                self.model_id,
                dtype=self.dtype,
            )
            .to(self.device)
            .<span class="hljs-built_in">eval</span>()
        )
        duration_ms = (time.perf_counter() - start) * <span class="hljs-number">1000</span>
        logger.info(<span class="hljs-string">f&quot;Finished loading <span class="hljs-subst">{self.model_id}</span> in <span class="hljs-subst">{duration_ms:<span class="hljs-number">.2</span>f}</span> ms&quot;</span>)

    <span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">unload</span>(<span class="hljs-params">self</span>):
        <span class="hljs-string">&quot;&quot;&quot;Free model + tokenizer and clear CUDA cache.&quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.model <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
            self.model.to(<span class="hljs-string">&quot;cpu&quot;</span>)
            <span class="hljs-keyword">del</span> self.model
            self.model = <span class="hljs-literal">None</span>

        <span class="hljs-keyword">if</span> self.tokenizer <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
            <span class="hljs-keyword">del</span> self.tokenizer
            self.tokenizer = <span class="hljs-literal">None</span>

        <span class="hljs-keyword">if</span> torch.cuda.is_available():
            torch.cuda.empty_cache()

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get</span>(<span class="hljs-params">self</span>):
        <span class="hljs-string">&quot;&quot;&quot;Return the loaded model + tokenizer or raise if not ready.&quot;&quot;&quot;</span>
        <span class="hljs-keyword">if</span> self.model <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> self.tokenizer <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
            <span class="hljs-keyword">raise</span> ModelNotLoadedError(<span class="hljs-string">&quot;Model not loaded&quot;</span>)
        <span class="hljs-keyword">return</span> self.model, self.tokenizer


model_manager = ModelManager(MODEL_ID, DEVICE, DTYPE)


<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Lifespan (startup + shutdown)</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-meta">@asynccontextmanager</span>
<span class="hljs-keyword">async</span> <span class="hljs-keyword">def</span> <span class="hljs-title function_">lifespan</span>(<span class="hljs-params">app: FastAPI</span>):
    <span class="hljs-keyword">await</span> model_manager.load()
    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">yield</span>
    <span class="hljs-keyword">finally</span>:
        <span class="hljs-keyword">await</span> model_manager.unload()


app = FastAPI(lifespan=lifespan)


<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Schemas</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">GenerateRequest</span>(<span class="hljs-title class_ inherited__">BaseModel</span>):
    prompt: <span class="hljs-built_in">str</span> = Field(..., min_length=<span class="hljs-number">1</span>, description=<span class="hljs-string">&quot;Plain-text prompt&quot;</span>)
    max_new_tokens: <span class="hljs-built_in">int</span> = Field(
        <span class="hljs-number">128</span>,
        ge=<span class="hljs-number">1</span>,
        le=MAX_NEW_TOKENS,
        description=<span class="hljs-string">&quot;Upper bound on generated tokens&quot;</span>,
    )


<span class="hljs-keyword">class</span> <span class="hljs-title class_">GenerateResponse</span>(<span class="hljs-title class_ inherited__">BaseModel</span>):
    response: <span class="hljs-built_in">str</span>
    input_token_count: <span class="hljs-built_in">int</span>
    output_token_count: <span class="hljs-built_in">int</span>


<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-comment"># Routes</span>
<span class="hljs-comment"># ------------------------------------------------------</span>
<span class="hljs-meta">@app.get(<span class="hljs-params"><span class="hljs-string">&quot;/health&quot;</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">health</span>():
    <span class="hljs-keyword">try</span>:
        model_manager.get()
    <span class="hljs-keyword">except</span> ModelNotLoadedError <span class="hljs-keyword">as</span> exc:
        <span class="hljs-keyword">raise</span> HTTPException(status_code=<span class="hljs-number">503</span>, detail=<span class="hljs-built_in">str</span>(exc)) <span class="hljs-keyword">from</span> exc
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;message&quot;</span>: <span class="hljs-string">&quot;API is running.&quot;</span>}


<span class="hljs-meta">@app.post(<span class="hljs-params"><span class="hljs-string">&quot;/generate&quot;</span>, response_model=GenerateResponse</span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate</span>(<span class="hljs-params">request: GenerateRequest</span>) -&gt; GenerateResponse:
    start_time = time.perf_counter()
    <span class="hljs-keyword">try</span>:
        model, tokenizer = model_manager.get()
    <span class="hljs-keyword">except</span> ModelNotLoadedError <span class="hljs-keyword">as</span> exc:
        <span class="hljs-keyword">raise</span> HTTPException(status_code=<span class="hljs-number">503</span>, detail=<span class="hljs-built_in">str</span>(exc)) <span class="hljs-keyword">from</span> exc

    <span class="hljs-keyword">if</span> <span class="hljs-built_in">getattr</span>(tokenizer, <span class="hljs-string">&quot;chat_template&quot;</span>, <span class="hljs-literal">None</span>):
        messages = [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: request.prompt}]
        input_text = tokenizer.apply_chat_template(
            messages,
            tokenize=<span class="hljs-literal">False</span>,
            add_generation_prompt=<span class="hljs-literal">True</span>,
        )
    <span class="hljs-keyword">else</span>:
        input_text = request.prompt

    inputs = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(DEVICE)

    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">with</span> torch.inference_mode():
            outputs = model.generate(**inputs, max_new_tokens=request.max_new_tokens)
    <span class="hljs-keyword">except</span> RuntimeError <span class="hljs-keyword">as</span> exc:
        logger.exception(<span class="hljs-string">&quot;Generation failed&quot;</span>)
        <span class="hljs-keyword">raise</span> HTTPException(
            status_code=<span class="hljs-number">500</span>, detail=<span class="hljs-string">f&quot;Generation failed: <span class="hljs-subst">{exc}</span>&quot;</span>
        ) <span class="hljs-keyword">from</span> exc

    input_token_count = inputs.input_ids.shape[<span class="hljs-number">1</span>]
    generated_ids = outputs[<span class="hljs-number">0</span>][input_token_count:]
    generated_text = tokenizer.decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)
    output_token_count = generated_ids.shape[<span class="hljs-number">0</span>]
    duration_ms = (time.perf_counter() - start_time) * <span class="hljs-number">1000</span>

    logger.info(
        <span class="hljs-string">f&quot;generate prompt_tokens=<span class="hljs-subst">{input_token_count}</span> &quot;</span>
        <span class="hljs-string">f&quot;new_tokens=<span class="hljs-subst">{output_token_count}</span> max_new_tokens=<span class="hljs-subst">{request.max_new_tokens}</span> &quot;</span>
        <span class="hljs-string">f&quot;duration_ms=<span class="hljs-subst">{duration_ms:<span class="hljs-number">.2</span>f}</span>&quot;</span>
    )

    <span class="hljs-keyword">return</span> GenerateResponse(
        response=generated_text,
        input_token_count=input_token_count,
        output_token_count=output_token_count,
    )`,wrap:!1}}),dl=new U({props:{title:"2. Build the Docker image",local:"2-build-the-docker-image",headingTag:"h2"}}),Il=new r({props:{code:"dXYlMjBsb2Nr",highlighted:"uv lock",wrap:!1}}),bl=new r({props:{code:"RlJPTSUyMHB5dG9yY2glMkZweXRvcmNoJTNBMi45LjEtY3VkYTEyLjgtY3Vkbm45LXJ1bnRpbWUlMEElMEElMjMlMjBJbnN0YWxsJTIwdXYlMjBieSUyMGNvcHlpbmclMjB0aGUlMjBzdGF0aWMlMjBiaW5hcnklMjBmcm9tJTIwdGhlJTIwZGlzdHJvbGVzcyUyMGltYWdlLiUwQUNPUFklMjAtLWZyb20lM0RnaGNyLmlvJTJGYXN0cmFsLXNoJTJGdXYlM0FsYXRlc3QlMjAlMkZ1diUyMCUyRmJpbiUyRnV2JTBBQ09QWSUyMC0tZnJvbSUzRGdoY3IuaW8lMkZhc3RyYWwtc2glMkZ1diUzQWxhdGVzdCUyMCUyRnV2eCUyMCUyRmJpbiUyRnV2eCUwQSUwQUVOViUyMFVTRVIlM0RhcHB1c2VyJTIwSE9NRSUzRCUyRmhvbWUlMkZhcHB1c2VyJTBBUlVOJTIwdXNlcmFkZCUyMC1tJTIwLXMlMjAlMkZiaW4lMkZiYXNoJTIwJTI0VVNFUiUwQSUwQVdPUktESVIlMjAlMkZhcHAlMEElMEElMjMlMjBFbnN1cmUlMjB1diUyMHVzZXMlMjB0aGUlMjBidW5kbGVkJTIwdmVudiUwQUVOViUyMFZJUlRVQUxfRU5WJTNEJTJGYXBwJTJGLnZlbnYlMEFFTlYlMjBQQVRIJTNEJTIyJTJGYXBwJTJGLnZlbnYlMkZiaW4lM0ElMjQlN0JQQVRIJTdEJTIyJTBBJTBBJTIzJTIwQ29weSUyMHByb2plY3QlMjBtZXRhZGF0YSUyMGZpcnN0JTIwKGJldHRlciUyMGNhY2hpbmcpJTBBQ09QWSUyMHB5cHJvamVjdC50b21sJTIwdXYubG9jayUyMC4lMkYlMEElMEElMjMlMjBDcmVhdGUlMjB0aGUlMjB2ZW52JTIwdXAlMjBmcm9udCUyMGFuZCUyMHN5bmMlMjBkZXBlbmRlbmNpZXMlMjAobm8lMjBzb3VyY2UlMjB5ZXQlMjBmb3IlMjBiZXR0ZXIlMjBjYWNoaW5nKSUwQVJVTiUyMHV2JTIwdmVudiUyMCUyNCU3QlZJUlRVQUxfRU5WJTdEJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTI2JTI2JTIwdXYlMjBzeW5jJTIwLS1mcm96ZW4lMjAtLW5vLWRldiUyMC0tbm8taW5zdGFsbC1wcm9qZWN0JTBBJTBBJTIzJTIwQ29weSUyMHRoZSUyMG1haW4lMjBhcHBsaWNhdGlvbiUyMGNvZGUlMEFDT1BZJTIwbWFpbi5weSUyMC4lMEElMEElMjMlMjBSZS1zeW5jJTIwdG8lMjBjYXB0dXJlJTIwdGhlJTIwcHJvamVjdCUyMGl0c2VsZiUyMGluc2lkZSUyMHRoZSUyMHZlbnYlMEFSVU4lMjB1diUyMHN5bmMlMjAtLWZyb3plbiUyMC0tbm8tZGV2JTBBJTBBUlVOJTIwY2hvd24lMjAtUiUyMCUyNFVTRVIlM0ElMjRVU0VSJTIwJTJGYXBwJTBBJTBBVVNFUiUyMCUyNFVTRVIlMEElMEFFWFBPU0UlMjA4MDAwJTBBJTBBQ01EJTIwJTVCJTIydXZpY29ybiUyMiUyQyUyMCUyMm1haW4lM0FhcHAlMjIlMkMlMjAlMjItLWhvc3QlMjIlMkMlMjAlMjIwLjAuMC4wJTIyJTJDJTIwJTIyLS1wb3J0JTIyJTJDJTIwJTIyODAwMCUyMiU1RA==",highlighted:`<span class="hljs-keyword">FROM</span> pytorch/pytorch:<span class="hljs-number">2.9</span>.<span class="hljs-number">1</span>-cuda12.<span class="hljs-number">8</span>-cudnn9-runtime

<span class="hljs-comment"># Install uv by copying the static binary from the distroless image.</span>
<span class="hljs-keyword">COPY</span><span class="language-bash"> --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv</span>
<span class="hljs-keyword">COPY</span><span class="language-bash"> --from=ghcr.io/astral-sh/uv:latest /uvx /bin/uvx</span>

<span class="hljs-keyword">ENV</span> <span class="hljs-keyword">USER</span>=appuser HOME=/home/appuser
<span class="hljs-keyword">RUN</span><span class="language-bash"> useradd -m -s /bin/bash <span class="hljs-variable">$USER</span></span>

<span class="hljs-keyword">WORKDIR</span><span class="language-bash"> /app</span>

<span class="hljs-comment"># Ensure uv uses the bundled venv</span>
<span class="hljs-keyword">ENV</span> VIRTUAL_ENV=/app/.venv
<span class="hljs-keyword">ENV</span> PATH=<span class="hljs-string">&quot;/app/.venv/bin:\${PATH}&quot;</span>

<span class="hljs-comment"># Copy project metadata first (better caching)</span>
<span class="hljs-keyword">COPY</span><span class="language-bash"> pyproject.toml uv.lock ./</span>

<span class="hljs-comment"># Create the venv up front and sync dependencies (no source yet for better caching)</span>
<span class="hljs-keyword">RUN</span><span class="language-bash"> uv venv <span class="hljs-variable">\${VIRTUAL_ENV}</span> \\
    &amp;&amp; uv <span class="hljs-built_in">sync</span> --frozen --no-dev --no-install-project</span>

<span class="hljs-comment"># Copy the main application code</span>
<span class="hljs-keyword">COPY</span><span class="language-bash"> main.py .</span>

<span class="hljs-comment"># Re-sync to capture the project itself inside the venv</span>
<span class="hljs-keyword">RUN</span><span class="language-bash"> uv <span class="hljs-built_in">sync</span> --frozen --no-dev</span>

<span class="hljs-keyword">RUN</span><span class="language-bash"> <span class="hljs-built_in">chown</span> -R <span class="hljs-variable">$USER</span>:<span class="hljs-variable">$USER</span> /app</span>

<span class="hljs-keyword">USER</span> $<span class="hljs-keyword">USER</span>

<span class="hljs-keyword">EXPOSE</span> <span class="hljs-number">8000</span>

<span class="hljs-keyword">CMD</span><span class="language-bash"> [<span class="hljs-string">&quot;uvicorn&quot;</span>, <span class="hljs-string">&quot;main:app&quot;</span>, <span class="hljs-string">&quot;--host&quot;</span>, <span class="hljs-string">&quot;0.0.0.0&quot;</span>, <span class="hljs-string">&quot;--port&quot;</span>, <span class="hljs-string">&quot;8000&quot;</span>]</span>`,wrap:!1}}),Bl=new U({props:{title:"3. Build and Push the Image",local:"3-build-and-push-the-image",headingTag:"h2"}}),fl=new r({props:{code:"ZG9ja2VyJTIwYnVpbGQlMjAtdCUyMHlvdXItdXNlcm5hbWUlMkZzbW9sbG0tZW5kcG9pbnQlM0F2MC4xLjAlMjAuJTIwLS1wbGF0Zm9ybSUyMGxpbnV4JTJGYW1kNjQlMEFkb2NrZXIlMjBwdXNoJTIweW91ci11c2VybmFtZSUyRnNtb2xsbS1lbmRwb2ludCUzQXYwLjEuMA==",highlighted:`docker build -t your-username/smollm-endpoint:v0.1.0 . --platform linux/amd64
docker push your-username/smollm-endpoint:v0.1.0`,wrap:!1}}),Zl=new U({props:{title:"4. Create the Endpoint",local:"4-create-the-endpoint",headingTag:"h2"}}),El=new r({props:{code:"aHR0cHMlM0ElMkYlMkZyYW5kb20tbnVtYmVyLnJlZ2lvbi5lbmRwb2ludHMuaHVnZ2luZ2ZhY2UuY2xvdWQlMkY=",highlighted:"https://random-number.region.endpoints.huggingface.cloud/",wrap:!1}}),Al=new r({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF90b2tlbiUwQWltcG9ydCUyMHJlcXVlc3RzJTBBJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZyYW5kb20tbnVtYmVyLnJlZ2lvbi5lbmRwb2ludHMuaHVnZ2luZ2ZhY2UuY2xvdWQlMkZnZW5lcmF0ZSUyMiUwQSUwQXByb21wdCUyMCUzRCUyMCUyMldoYXQlMjBpcyUyMGFuJTIwSW5mZXJlbmNlJTIwRW5kcG9pbnQlM0YlMjIlMEFkYXRhJTIwJTNEJTIwJTdCJTIycHJvbXB0JTIyJTNBJTIwcHJvbXB0JTJDJTIwJTIybWF4X25ld190b2tlbnMlMjIlM0ElMjA1MTIlN0QlMEElMEElMEFyZXNwb25zZSUyMCUzRCUyMHJlcXVlc3RzLnBvc3QoJTBBJTIwJTIwJTIwJTIwdXJsJTNEdXJsJTJDJTBBJTIwJTIwJTIwJTIwanNvbiUzRGRhdGElMkMlMEElMjAlMjAlMjAlMjBoZWFkZXJzJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyQXV0aG9yaXphdGlvbiUyMiUzQSUyMGYlMjJCZWFyZXIlMjAlN0JnZXRfdG9rZW4oKSU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMkNvbnRlbnQtVHlwZSUyMiUzQSUyMCUyMmFwcGxpY2F0aW9uJTJGanNvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSkuanNvbigpJTBBJTBBcHJpbnQoZiUyMklucHV0JTNBJTVDbiU3QnByb21wdCU3RCU1Q24lNUNuT3V0cHV0JTNBJTVDbiU3QnJlc3BvbnNlJTVCJ3Jlc3BvbnNlJyU1RCU3RCUyMiklMEE=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_token
<span class="hljs-keyword">import</span> requests

url = <span class="hljs-string">&quot;https://random-number.region.endpoints.huggingface.cloud/generate&quot;</span>

prompt = <span class="hljs-string">&quot;What is an Inference Endpoint?&quot;</span>
data = {<span class="hljs-string">&quot;prompt&quot;</span>: prompt, <span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">512</span>}


response = requests.post(
    url=url,
    json=data,
    headers={
        <span class="hljs-string">&quot;Authorization&quot;</span>: <span class="hljs-string">f&quot;Bearer <span class="hljs-subst">{get_token()}</span>&quot;</span>,
        <span class="hljs-string">&quot;Content-Type&quot;</span>: <span class="hljs-string">&quot;application/json&quot;</span>,
    },
).json()

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Input:\\n<span class="hljs-subst">{prompt}</span>\\n\\nOutput:\\n<span class="hljs-subst">{response[<span class="hljs-string">&#x27;response&#x27;</span>]}</span>&quot;</span>)
`,wrap:!1}}),vl=new r({props:{code:"SW5wdXQlM0ElMEFXaGF0JTIwaXMlMjBhbiUyMEluZmVyZW5jZSUyMEVuZHBvaW50JTNGJTBBJTBBT3V0cHV0JTNBJTBBJTNDdGhpbmslM0UlMEFPa2F5JTJDJTIwc28lMjBJJTIwbmVlZCUyMHRvJTIwLi4u",highlighted:`<span class="hljs-symbol">Input:</span>
What <span class="hljs-built_in">is</span> an Inference Endpoint?

<span class="hljs-symbol">Output:</span>
&lt;think&gt;
Okay, so I need <span class="hljs-keyword">to</span> ...`,wrap:!1}}),Rl=new U({props:{title:"5. Next steps and extensions",local:"5-next-steps-and-extensions",headingTag:"h2"}}),_l=new Ct({props:{source:"https://github.com/huggingface/hf-endpoints-documentation/blob/main/docs/source/engines/custom_container.md"}}),{c(){d=M("meta"),zl=n(),Yl=M("p"),$l=n(),y(b.$$.fragment),ql=n(),y(B.$$.fragment),Dl=n(),S=M("p"),S.innerHTML=Be,Pl=n(),f=M("p"),f.textContent=Se,Kl=n(),Z=M("p"),Z.textContent=fe,Ol=n(),g=M("ul"),g.innerHTML=Ze,ls=n(),k=M("p"),k.textContent=ge,ss=n(),y(L.$$.fragment),es=n(),y(E.$$.fragment),ts=n(),G=M("p"),G.textContent=ke,ns=n(),y(A.$$.fragment),as=n(),j=M("blockquote"),j.innerHTML=Le,Ms=n(),V=M("p"),V.innerHTML=Ee,ps=n(),W=M("ul"),W.innerHTML=Ge,os=n(),u=M("blockquote"),u.innerHTML=Ae,ys=n(),y(v.$$.fragment),is=n(),R=M("p"),R.textContent=Ve,ws=n(),y(X.$$.fragment),Js=n(),N=M("p"),N.innerHTML=We,Ts=n(),y(Q.$$.fragment),cs=n(),y(_.$$.fragment),rs=n(),y(x.$$.fragment),Us=n(),F=M("p"),F.textContent=ve,ds=n(),Y=M("ol"),Y.innerHTML=Re,js=n(),y(H.$$.fragment),us=n(),y(z.$$.fragment),ms=n(),m=M("ol"),m.innerHTML=Xe,Is=n(),y($.$$.fragment),hs=n(),y(q.$$.fragment),Cs=n(),D=M("p"),D.textContent=Ne,bs=n(),P=M("p"),P.innerHTML=Qe,Bs=n(),y(K.$$.fragment),Ss=n(),O=M("p"),O.innerHTML=_e,fs=n(),y(ll.$$.fragment),Zs=n(),sl=M("p"),sl.innerHTML=xe,gs=n(),el=M("p"),el.innerHTML=Fe,ks=n(),y(tl.$$.fragment),Ls=n(),nl=M("p"),nl.innerHTML=Ye,Es=n(),al=M("ul"),al.innerHTML=He,Gs=n(),y(Ml.$$.fragment),As=n(),y(pl.$$.fragment),Vs=n(),ol=M("p"),ol.textContent=ze,Ws=n(),y(yl.$$.fragment),vs=n(),il=M("p"),il.textContent=$e,Rs=n(),wl=M("p"),wl.textContent=qe,Xs=n(),y(Jl.$$.fragment),Ns=n(),Tl=M("p"),Tl.innerHTML=De,Qs=n(),cl=M("p"),cl.textContent=Pe,_s=n(),y(rl.$$.fragment),xs=n(),I=M("details"),Fl=M("summary"),Fl.textContent=Ke,Ie=n(),y(Ul.$$.fragment),Fs=n(),y(dl.$$.fragment),Ys=n(),jl=M("p"),jl.innerHTML=Oe,Hs=n(),h=M("blockquote"),h.innerHTML=lt,zs=n(),ul=M("p"),ul.innerHTML=st,$s=n(),ml=M("p"),ml.textContent=et,qs=n(),y(Il.$$.fragment),Ds=n(),hl=M("p"),hl.textContent=tt,Ps=n(),Cl=M("ol"),Cl.innerHTML=nt,Ks=n(),y(bl.$$.fragment),Os=n(),y(Bl.$$.fragment),le=n(),Sl=M("p"),Sl.innerHTML=at,se=n(),y(fl.$$.fragment),ee=n(),C=M("blockquote"),C.innerHTML=Mt,te=n(),y(Zl.$$.fragment),ne=n(),gl=M("p"),gl.textContent=pt,ae=n(),kl=M("ol"),kl.innerHTML=ot,Me=n(),Ll=M("p"),Ll.textContent=yt,pe=n(),y(El.$$.fragment),oe=n(),Gl=M("p"),Gl.textContent=it,ye=n(),y(Al.$$.fragment),ie=n(),Vl=M("p"),Vl.innerHTML=wt,we=n(),Wl=M("p"),Wl.innerHTML=Jt,Je=n(),y(vl.$$.fragment),Te=n(),y(Rl.$$.fragment),ce=n(),Xl=M("p"),Xl.textContent=Tt,re=n(),Nl=M("p"),Nl.textContent=ct,Ue=n(),Ql=M("p"),Ql.textContent=rt,de=n(),y(_l.$$.fragment),je=n(),Hl=M("p"),this.h()},l(l){const s=It("svelte-u9bgzb",document.head);d=p(s,"META",{name:!0,content:!0}),s.forEach(e),zl=a(l),Yl=p(l,"P",{}),he(Yl).forEach(e),$l=a(l),i(b.$$.fragment,l),ql=a(l),i(B.$$.fragment,l),Dl=a(l),S=p(l,"P",{"data-svelte-h":!0}),o(S)!=="svelte-h6ujrw"&&(S.innerHTML=Be),Pl=a(l),f=p(l,"P",{"data-svelte-h":!0}),o(f)!=="svelte-1lht9kg"&&(f.textContent=Se),Kl=a(l),Z=p(l,"P",{"data-svelte-h":!0}),o(Z)!=="svelte-u7wl0r"&&(Z.textContent=fe),Ol=a(l),g=p(l,"UL",{"data-svelte-h":!0}),o(g)!=="svelte-15os693"&&(g.innerHTML=Ze),ls=a(l),k=p(l,"P",{"data-svelte-h":!0}),o(k)!=="svelte-jr240p"&&(k.textContent=ge),ss=a(l),i(L.$$.fragment,l),es=a(l),i(E.$$.fragment,l),ts=a(l),G=p(l,"P",{"data-svelte-h":!0}),o(G)!=="svelte-12ix9m7"&&(G.textContent=ke),ns=a(l),i(A.$$.fragment,l),as=a(l),j=p(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(j)!=="svelte-icy6tp"&&(j.innerHTML=Le),Ms=a(l),V=p(l,"P",{"data-svelte-h":!0}),o(V)!=="svelte-zhi31g"&&(V.innerHTML=Ee),ps=a(l),W=p(l,"UL",{"data-svelte-h":!0}),o(W)!=="svelte-1kqxuj4"&&(W.innerHTML=Ge),os=a(l),u=p(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(u)!=="svelte-wanl1l"&&(u.innerHTML=Ae),ys=a(l),i(v.$$.fragment,l),is=a(l),R=p(l,"P",{"data-svelte-h":!0}),o(R)!=="svelte-qy05hr"&&(R.textContent=Ve),ws=a(l),i(X.$$.fragment,l),Js=a(l),N=p(l,"P",{"data-svelte-h":!0}),o(N)!=="svelte-1jom1yq"&&(N.innerHTML=We),Ts=a(l),i(Q.$$.fragment,l),cs=a(l),i(_.$$.fragment,l),rs=a(l),i(x.$$.fragment,l),Us=a(l),F=p(l,"P",{"data-svelte-h":!0}),o(F)!=="svelte-1wscg1i"&&(F.textContent=ve),ds=a(l),Y=p(l,"OL",{"data-svelte-h":!0}),o(Y)!=="svelte-1g3514s"&&(Y.innerHTML=Re),js=a(l),i(H.$$.fragment,l),us=a(l),i(z.$$.fragment,l),ms=a(l),m=p(l,"OL",{start:!0,"data-svelte-h":!0}),o(m)!=="svelte-6k8ggt"&&(m.innerHTML=Xe),Is=a(l),i($.$$.fragment,l),hs=a(l),i(q.$$.fragment,l),Cs=a(l),D=p(l,"P",{"data-svelte-h":!0}),o(D)!=="svelte-k9psp2"&&(D.textContent=Ne),bs=a(l),P=p(l,"P",{"data-svelte-h":!0}),o(P)!=="svelte-16wcsb8"&&(P.innerHTML=Qe),Bs=a(l),i(K.$$.fragment,l),Ss=a(l),O=p(l,"P",{"data-svelte-h":!0}),o(O)!=="svelte-lqc8e6"&&(O.innerHTML=_e),fs=a(l),i(ll.$$.fragment,l),Zs=a(l),sl=p(l,"P",{"data-svelte-h":!0}),o(sl)!=="svelte-1tiw1xa"&&(sl.innerHTML=xe),gs=a(l),el=p(l,"P",{"data-svelte-h":!0}),o(el)!=="svelte-2isapa"&&(el.innerHTML=Fe),ks=a(l),i(tl.$$.fragment,l),Ls=a(l),nl=p(l,"P",{"data-svelte-h":!0}),o(nl)!=="svelte-qoor7x"&&(nl.innerHTML=Ye),Es=a(l),al=p(l,"UL",{"data-svelte-h":!0}),o(al)!=="svelte-1penqq8"&&(al.innerHTML=He),Gs=a(l),i(Ml.$$.fragment,l),As=a(l),i(pl.$$.fragment,l),Vs=a(l),ol=p(l,"P",{"data-svelte-h":!0}),o(ol)!=="svelte-b277l5"&&(ol.textContent=ze),Ws=a(l),i(yl.$$.fragment,l),vs=a(l),il=p(l,"P",{"data-svelte-h":!0}),o(il)!=="svelte-eeafha"&&(il.textContent=$e),Rs=a(l),wl=p(l,"P",{"data-svelte-h":!0}),o(wl)!=="svelte-18opxl"&&(wl.textContent=qe),Xs=a(l),i(Jl.$$.fragment,l),Ns=a(l),Tl=p(l,"P",{"data-svelte-h":!0}),o(Tl)!=="svelte-1tl1lae"&&(Tl.innerHTML=De),Qs=a(l),cl=p(l,"P",{"data-svelte-h":!0}),o(cl)!=="svelte-18rlkl3"&&(cl.textContent=Pe),_s=a(l),i(rl.$$.fragment,l),xs=a(l),I=p(l,"DETAILS",{});var me=he(I);Fl=p(me,"SUMMARY",{"data-svelte-h":!0}),o(Fl)!=="svelte-1necd30"&&(Fl.textContent=Ke),Ie=a(me),i(Ul.$$.fragment,me),me.forEach(e),Fs=a(l),i(dl.$$.fragment,l),Ys=a(l),jl=p(l,"P",{"data-svelte-h":!0}),o(jl)!=="svelte-zvx9kj"&&(jl.innerHTML=Oe),Hs=a(l),h=p(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(h)!=="svelte-12w37im"&&(h.innerHTML=lt),zs=a(l),ul=p(l,"P",{"data-svelte-h":!0}),o(ul)!=="svelte-n5vhv9"&&(ul.innerHTML=st),$s=a(l),ml=p(l,"P",{"data-svelte-h":!0}),o(ml)!=="svelte-2hy6e"&&(ml.textContent=et),qs=a(l),i(Il.$$.fragment,l),Ds=a(l),hl=p(l,"P",{"data-svelte-h":!0}),o(hl)!=="svelte-1mplbnu"&&(hl.textContent=tt),Ps=a(l),Cl=p(l,"OL",{"data-svelte-h":!0}),o(Cl)!=="svelte-1vepoie"&&(Cl.innerHTML=nt),Ks=a(l),i(bl.$$.fragment,l),Os=a(l),i(Bl.$$.fragment,l),le=a(l),Sl=p(l,"P",{"data-svelte-h":!0}),o(Sl)!=="svelte-1le3hym"&&(Sl.innerHTML=at),se=a(l),i(fl.$$.fragment,l),ee=a(l),C=p(l,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(C)!=="svelte-nzg1pu"&&(C.innerHTML=Mt),te=a(l),i(Zl.$$.fragment,l),ne=a(l),gl=p(l,"P",{"data-svelte-h":!0}),o(gl)!=="svelte-79a9ta"&&(gl.textContent=pt),ae=a(l),kl=p(l,"OL",{"data-svelte-h":!0}),o(kl)!=="svelte-1rj27o6"&&(kl.innerHTML=ot),Me=a(l),Ll=p(l,"P",{"data-svelte-h":!0}),o(Ll)!=="svelte-1ur9euj"&&(Ll.textContent=yt),pe=a(l),i(El.$$.fragment,l),oe=a(l),Gl=p(l,"P",{"data-svelte-h":!0}),o(Gl)!=="svelte-1w5htx2"&&(Gl.textContent=it),ye=a(l),i(Al.$$.fragment,l),ie=a(l),Vl=p(l,"P",{"data-svelte-h":!0}),o(Vl)!=="svelte-w6xt9s"&&(Vl.innerHTML=wt),we=a(l),Wl=p(l,"P",{"data-svelte-h":!0}),o(Wl)!=="svelte-jvs2q"&&(Wl.innerHTML=Jt),Je=a(l),i(vl.$$.fragment,l),Te=a(l),i(Rl.$$.fragment,l),ce=a(l),Xl=p(l,"P",{"data-svelte-h":!0}),o(Xl)!=="svelte-dqwtkl"&&(Xl.textContent=Tt),re=a(l),Nl=p(l,"P",{"data-svelte-h":!0}),o(Nl)!=="svelte-1ynngyx"&&(Nl.textContent=ct),Ue=a(l),Ql=p(l,"P",{"data-svelte-h":!0}),o(Ql)!=="svelte-1ueg8ci"&&(Ql.textContent=rt),de=a(l),i(_l.$$.fragment,l),je=a(l),Hl=p(l,"P",{}),he(Hl).forEach(e),this.h()},h(){xl(d,"name","hf:doc:metadata"),xl(d,"content",Bt),xl(j,"class","note"),xl(u,"class","important"),xl(m,"start","2"),xl(h,"class","tip"),xl(C,"class","note")},m(l,s){Ce(document.head,d),t(l,zl,s),t(l,Yl,s),t(l,$l,s),w(b,l,s),t(l,ql,s),w(B,l,s),t(l,Dl,s),t(l,S,s),t(l,Pl,s),t(l,f,s),t(l,Kl,s),t(l,Z,s),t(l,Ol,s),t(l,g,s),t(l,ls,s),t(l,k,s),t(l,ss,s),w(L,l,s),t(l,es,s),w(E,l,s),t(l,ts,s),t(l,G,s),t(l,ns,s),w(A,l,s),t(l,as,s),t(l,j,s),t(l,Ms,s),t(l,V,s),t(l,ps,s),t(l,W,s),t(l,os,s),t(l,u,s),t(l,ys,s),w(v,l,s),t(l,is,s),t(l,R,s),t(l,ws,s),w(X,l,s),t(l,Js,s),t(l,N,s),t(l,Ts,s),w(Q,l,s),t(l,cs,s),w(_,l,s),t(l,rs,s),w(x,l,s),t(l,Us,s),t(l,F,s),t(l,ds,s),t(l,Y,s),t(l,js,s),w(H,l,s),t(l,us,s),w(z,l,s),t(l,ms,s),t(l,m,s),t(l,Is,s),w($,l,s),t(l,hs,s),w(q,l,s),t(l,Cs,s),t(l,D,s),t(l,bs,s),t(l,P,s),t(l,Bs,s),w(K,l,s),t(l,Ss,s),t(l,O,s),t(l,fs,s),w(ll,l,s),t(l,Zs,s),t(l,sl,s),t(l,gs,s),t(l,el,s),t(l,ks,s),w(tl,l,s),t(l,Ls,s),t(l,nl,s),t(l,Es,s),t(l,al,s),t(l,Gs,s),w(Ml,l,s),t(l,As,s),w(pl,l,s),t(l,Vs,s),t(l,ol,s),t(l,Ws,s),w(yl,l,s),t(l,vs,s),t(l,il,s),t(l,Rs,s),t(l,wl,s),t(l,Xs,s),w(Jl,l,s),t(l,Ns,s),t(l,Tl,s),t(l,Qs,s),t(l,cl,s),t(l,_s,s),w(rl,l,s),t(l,xs,s),t(l,I,s),Ce(I,Fl),Ce(I,Ie),w(Ul,I,null),t(l,Fs,s),w(dl,l,s),t(l,Ys,s),t(l,jl,s),t(l,Hs,s),t(l,h,s),t(l,zs,s),t(l,ul,s),t(l,$s,s),t(l,ml,s),t(l,qs,s),w(Il,l,s),t(l,Ds,s),t(l,hl,s),t(l,Ps,s),t(l,Cl,s),t(l,Ks,s),w(bl,l,s),t(l,Os,s),w(Bl,l,s),t(l,le,s),t(l,Sl,s),t(l,se,s),w(fl,l,s),t(l,ee,s),t(l,C,s),t(l,te,s),w(Zl,l,s),t(l,ne,s),t(l,gl,s),t(l,ae,s),t(l,kl,s),t(l,Me,s),t(l,Ll,s),t(l,pe,s),w(El,l,s),t(l,oe,s),t(l,Gl,s),t(l,ye,s),w(Al,l,s),t(l,ie,s),t(l,Vl,s),t(l,we,s),t(l,Wl,s),t(l,Je,s),w(vl,l,s),t(l,Te,s),w(Rl,l,s),t(l,ce,s),t(l,Xl,s),t(l,re,s),t(l,Nl,s),t(l,Ue,s),t(l,Ql,s),t(l,de,s),w(_l,l,s),t(l,je,s),t(l,Hl,s),ue=!0},p:dt,i(l){ue||(J(b.$$.fragment,l),J(B.$$.fragment,l),J(L.$$.fragment,l),J(E.$$.fragment,l),J(A.$$.fragment,l),J(v.$$.fragment,l),J(X.$$.fragment,l),J(Q.$$.fragment,l),J(_.$$.fragment,l),J(x.$$.fragment,l),J(H.$$.fragment,l),J(z.$$.fragment,l),J($.$$.fragment,l),J(q.$$.fragment,l),J(K.$$.fragment,l),J(ll.$$.fragment,l),J(tl.$$.fragment,l),J(Ml.$$.fragment,l),J(pl.$$.fragment,l),J(yl.$$.fragment,l),J(Jl.$$.fragment,l),J(rl.$$.fragment,l),J(Ul.$$.fragment,l),J(dl.$$.fragment,l),J(Il.$$.fragment,l),J(bl.$$.fragment,l),J(Bl.$$.fragment,l),J(fl.$$.fragment,l),J(Zl.$$.fragment,l),J(El.$$.fragment,l),J(Al.$$.fragment,l),J(vl.$$.fragment,l),J(Rl.$$.fragment,l),J(_l.$$.fragment,l),ue=!0)},o(l){T(b.$$.fragment,l),T(B.$$.fragment,l),T(L.$$.fragment,l),T(E.$$.fragment,l),T(A.$$.fragment,l),T(v.$$.fragment,l),T(X.$$.fragment,l),T(Q.$$.fragment,l),T(_.$$.fragment,l),T(x.$$.fragment,l),T(H.$$.fragment,l),T(z.$$.fragment,l),T($.$$.fragment,l),T(q.$$.fragment,l),T(K.$$.fragment,l),T(ll.$$.fragment,l),T(tl.$$.fragment,l),T(Ml.$$.fragment,l),T(pl.$$.fragment,l),T(yl.$$.fragment,l),T(Jl.$$.fragment,l),T(rl.$$.fragment,l),T(Ul.$$.fragment,l),T(dl.$$.fragment,l),T(Il.$$.fragment,l),T(bl.$$.fragment,l),T(Bl.$$.fragment,l),T(fl.$$.fragment,l),T(Zl.$$.fragment,l),T(El.$$.fragment,l),T(Al.$$.fragment,l),T(vl.$$.fragment,l),T(Rl.$$.fragment,l),T(_l.$$.fragment,l),ue=!1},d(l){l&&(e(zl),e(Yl),e($l),e(ql),e(Dl),e(S),e(Pl),e(f),e(Kl),e(Z),e(Ol),e(g),e(ls),e(k),e(ss),e(es),e(ts),e(G),e(ns),e(as),e(j),e(Ms),e(V),e(ps),e(W),e(os),e(u),e(ys),e(is),e(R),e(ws),e(Js),e(N),e(Ts),e(cs),e(rs),e(Us),e(F),e(ds),e(Y),e(js),e(us),e(ms),e(m),e(Is),e(hs),e(Cs),e(D),e(bs),e(P),e(Bs),e(Ss),e(O),e(fs),e(Zs),e(sl),e(gs),e(el),e(ks),e(Ls),e(nl),e(Es),e(al),e(Gs),e(As),e(Vs),e(ol),e(Ws),e(vs),e(il),e(Rs),e(wl),e(Xs),e(Ns),e(Tl),e(Qs),e(cl),e(_s),e(xs),e(I),e(Fs),e(Ys),e(jl),e(Hs),e(h),e(zs),e(ul),e($s),e(ml),e(qs),e(Ds),e(hl),e(Ps),e(Cl),e(Ks),e(Os),e(le),e(Sl),e(se),e(ee),e(C),e(te),e(ne),e(gl),e(ae),e(kl),e(Me),e(Ll),e(pe),e(oe),e(Gl),e(ye),e(ie),e(Vl),e(we),e(Wl),e(Je),e(Te),e(ce),e(Xl),e(re),e(Nl),e(Ue),e(Ql),e(de),e(je),e(Hl)),e(d),c(b,l),c(B,l),c(L,l),c(E,l),c(A,l),c(v,l),c(X,l),c(Q,l),c(_,l),c(x,l),c(H,l),c(z,l),c($,l),c(q,l),c(K,l),c(ll,l),c(tl,l),c(Ml,l),c(pl,l),c(yl,l),c(Jl,l),c(rl,l),c(Ul),c(dl,l),c(Il,l),c(bl,l),c(Bl,l),c(fl,l),c(Zl,l),c(El,l),c(Al,l),c(vl,l),c(Rl,l),c(_l,l)}}}const Bt='{"title":"Deploy with your own container","local":"deploy-with-your-own-container","sections":[{"title":"1. Create the inference server","local":"1-create-the-inference-server","sections":[{"title":"1.1 Initialize the uv project","local":"11-initialize-the-uv-project","sections":[],"depth":3},{"title":"1.2 Install the Python dependencies","local":"12-install-the-python-dependencies","sections":[],"depth":3},{"title":"1.3 Add configurations","local":"13-add-configurations","sections":[],"depth":3},{"title":"1.4 Implement the ModelManager","local":"14-implement-the-modelmanager","sections":[],"depth":3},{"title":"1.5 Use FastAPI lifespan for startup and shutdown","local":"15-use-fastapi-lifespan-for-startup-and-shutdown","sections":[],"depth":3},{"title":"1.6 Define the request and response schemas","local":"16-define-the-request-and-response-schemas","sections":[],"depth":3},{"title":"1.7 Implement the server routes","local":"17-implement-the-server-routes","sections":[],"depth":3},{"title":"1.8 Run the server locally","local":"18-run-the-server-locally","sections":[],"depth":3},{"title":"1.9 Full server code listing","local":"19-full-server-code-listing","sections":[],"depth":3}],"depth":2},{"title":"2. Build the Docker image","local":"2-build-the-docker-image","sections":[],"depth":2},{"title":"3. Build and Push the Image","local":"3-build-and-push-the-image","sections":[],"depth":2},{"title":"4. Create the Endpoint","local":"4-create-the-endpoint","sections":[],"depth":2},{"title":"5. Next steps and extensions","local":"5-next-steps-and-extensions","sections":[],"depth":2}],"depth":1}';function St(be){return jt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Lt extends ut{constructor(d){super(),mt(this,d,St,bt,Ut,{})}}export{Lt as component};
