File size: 2,915 Bytes
1e3b872 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | async function* completion (url, messages, controller) {
let data = {
model: 'gpt-3.5-turbo-16k',
messages,
temperature: 0.05,
stream: true
}
// if (imageNode) {
// data = { ...data, image_data: [imageNode] }
// }
// let controller = new AbortController()
let response = await fetch(url, {
method: 'POST',
body: JSON.stringify(data),
headers: {
Connection: 'keep-alive',
'Content-Type': 'application/json',
Accept: 'text/event-stream'
},
signal: controller.signal
})
const reader = response.body.getReader()
const decoder = new TextDecoder()
let content = ''
let leftover = '' // Buffer for partially read lines
try {
let cont = true
while (cont) {
let result = await reader.read()
if (result.done) {
break
}
// Add any leftover data to the current chunk of data
const text = leftover + decoder.decode(result.value)
// Check if the last character is a line break
const endsWithLineBreak = text.endsWith('\n')
// Split the text into lines
let lines = text.split('\n')
// If the text doesn't end with a line break, then the last line is incomplete
// Store it in leftover to be added to the next chunk of data
if (!endsWithLineBreak) {
leftover = lines.pop()
} else {
leftover = '' // Reset leftover if we have a line break at the end
}
// Parse all sse events and add them to result
const regex = /^(\S+):\s(.*)$/gm
for (const line of lines) {
const match = regex.exec(line)
if (match) {
result[match[1]] = match[2]
// since we know this is llama.cpp, let's just decode the json in data
if (result.data) {
result.data = JSON.parse(result.data)
// console.log('#result.data',result.data)
content += result.data.choices[0].delta?.content || ''
// yield
yield result
// if we got a stop token from server, we will break here
if (result.data.choices[0].finish_reason == 'stop') {
if (result.data.generation_settings) {
// generation_settings = result.data.generation_settings;
}
cont = false
break
}
}
}
}
}
} catch (e) {
console.error('llama error: ', e)
throw e
} finally {
controller.abort()
}
return content
// return (await response.json()).content
}
export async function completion_ (url, messages, controller, callback) {
let request = await completion(url, messages, controller)
for await (const chunk of request) {
let content = chunk.data.choices[0].delta.content || ''
if (chunk.data.choices[0].role == 'assistant') {
//开始
content = ''
}
if (callback) callback(content)
}
}
|