File size: 2,637 Bytes
055eba4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | // Copyright 2024 Google LLC
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
#include "compression/io.h"
#include "gemma/benchmark_helper.h"
#include "gemma/gemma.h" // LayersOutputFunc
#include "util/args.h"
#include "hwy/base.h"
#include "nlohmann/json.hpp"
using json = nlohmann::json;
namespace gcpp {
class PromptArgs : public ArgsBase<PromptArgs> {
public:
PromptArgs(int argc, char* argv[]) { InitAndParse(argc, argv); }
Path layers_output; // optional
std::string prompt;
// Returns error string or nullptr if OK.
const char* Validate() const {
if (prompt.empty()) return "Must specify --prompt";
return nullptr;
}
template <class Visitor>
void ForEach(const Visitor& visitor) {
visitor(layers_output, "layers_output", Path(""),
"Path to store layers output", 2);
visitor(prompt, "prompt", std::string(""), "Prompt to the model", 2);
}
};
int Run(int argc, char** argv) {
PromptArgs prompt_args(argc, argv);
AbortIfInvalidArgs(prompt_args);
json json_output;
GemmaEnv env(argc, argv);
env.MutableConfig().layers_output =
prompt_args.layers_output.Empty()
? LayersOutputFunc()
: [&json_output](int pos, const std::string& key, const float* values,
size_t values_len) {
std::vector<float> v{values, values + values_len};
json_output[std::to_string(pos)][key] = v;
};
const auto [answer, token_count] = env.QueryModel(prompt_args.prompt);
std::cout << answer.substr(prompt_args.prompt.size()) << "\n" << std::flush;
if (env.MutableConfig().layers_output) {
std::ofstream output_f(prompt_args.layers_output.path, std::ofstream::out);
if (!output_f) HWY_ABORT("Opening layer output file failed");
output_f << json_output.dump();
if (!output_f) HWY_ABORT("Writing to layer output file failed");
output_f.close();
}
return 0;
}
} // namespace gcpp
int main(int argc, char** argv) { return gcpp::Run(argc, argv); }
|