Spaces:
Build error
Build error
File size: 1,078 Bytes
9db8ced 6cfb775 9db8ced 0e5c445 9db8ced dd4c436 9db8ced |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
import { HF_ACCESS_TOKEN } from "$env/static/private";
import { buildPrompt } from "$lib/buildPrompt";
import { textGenerationStream } from "@huggingface/inference";
import type { Endpoint } from "../endpoints";
import { z } from "zod";
export const endpointTgiParametersSchema = z.object({
weight: z.number().int().positive().default(1),
model: z.any(),
type: z.literal("tgi"),
url: z.string().url(),
accessToken: z.string().default(HF_ACCESS_TOKEN),
});
export function endpointTgi({
url,
accessToken,
model,
}: z.infer<typeof endpointTgiParametersSchema>): Endpoint {
return async ({ conversation }) => {
const prompt = await buildPrompt({
messages: conversation.messages,
webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
preprompt: conversation.preprompt,
model,
id: conversation._id,
});
return textGenerationStream(
{
parameters: { ...model.parameters, return_full_text: false },
model: url,
inputs: prompt,
accessToken,
},
{ use_cache: false }
);
};
}
export default endpointTgi;
|