File size: 1,768 Bytes
4a28d4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
#include "src/turbomind/engine/request.h"
#include <iterator>
namespace turbomind {
namespace {
template<typename T>
inline std::ostream& operator<<(std::ostream& os, const std::vector<T>& vec)
{
os << "[";
std::copy(vec.begin(), vec.end(), std::ostream_iterator<T>(os, ", "));
if (!vec.empty()) {
os.seekp(-2, std::ios_base::end);
}
os << "]";
return os;
}
} // namespace
std::ostream& operator<<(std::ostream& os, const GenerationConfig& c)
{
os << "GenerationConfig { ";
os << "max_new_tokens=" << c.max_new_tokens;
os << ", min_new_tokens=" << c.min_new_tokens;
os << ", eos_ids=" << c.eos_ids;
os << ", stop_ids=[" << c.stop_ids[0] << ", " << c.stop_ids[1] << "]";
os << ", bad_ids=[" << c.bad_ids[0] << ", " << c.bad_ids[1] << "]";
os << ", top_p=" << c.top_p;
os << ", top_k=" << c.top_k;
os << ", min_p=" << c.min_p;
os << ", temperature=" << c.temperature;
os << ", repetition_penalty=" << c.repetition_penalty;
os << ", random_seed=" << c.random_seed;
os << ", output_logprobs=" << c.output_logprobs;
os << ", output_hidden_states=" << c.output_last_hidden_state;
os << ", output_logits=" << c.output_logits;
os << " }";
return os;
}
void UpdateState(Request& r, int status, int seq_len)
{
try {
auto new_state = new RequestState{status, seq_len};
auto old_state = r.state->exchange(new_state);
if (!old_state && r.forward_cb) {
r.forward_cb();
}
}
catch (const std::exception& e) {
TM_LOG_ERROR("Error invoking callback for ({}): {}", r.id, e.what());
}
catch (...) {
TM_LOG_ERROR("Unknown error invoking callback for ({})", r.id);
}
}
} // namespace turbomind
|