File size: 1,058 Bytes
4a28d4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | // Copyright (c) OpenMMLab. All rights reserved.
#include <memory>
#include "src/turbomind/core/core.h"
#include "src/turbomind/engine/batch.h"
#include "src/turbomind/engine/queue.h"
#include "src/turbomind/models/language_model.h"
#include "src/turbomind/models/llama/context.h"
namespace turbomind {
// Model executor for auto-regressive language models
class ModelExecutor {
public:
~ModelExecutor();
ModelExecutor();
ModelExecutor(ModelExecutor&&) noexcept;
ModelExecutor& operator=(ModelExecutor&&) noexcept;
explicit operator bool() const noexcept
{
return static_cast<bool>(impl_);
}
ModelExecutor(LanguageModel& model,
Context& context,
int device_id,
Queue<std::unique_ptr<BatchData>>& inbound,
Queue<std::unique_ptr<BatchData>>& outbound);
void Start();
private:
struct Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace turbomind
|