YARP
Yet Another Robot Platform
 
Loading...
Searching...
No Matches
Llama2Device.cpp
Go to the documentation of this file.
1/*
2 * SPDX-FileCopyrightText: 2023 Istituto Italiano di Tecnologia (IIT)
3 * SPDX-License-Identifier: BSD-3-Clause
4 */
5
6#ifndef _USE_MATH_DEFINES
7#define _USE_MATH_DEFINES
8#endif
9
10#include "llama.h"
11#include "Llama2Device.h"
12#include <fstream>
14#include <yarp/os/LogStream.h>
16#include "common.h"
17#include "arg.h"
18#include "sampling.h"
19
20#include <cmath>
21#include <cstdio>
22#include <string>
23#include <vector>
24
25#include "Utils.h"
26
27using namespace yarp::dev;
28
30
35
37{
38 if (!parseParams(config)) { return false; }
39 // initialize LLM
40 std::string model_path;
41 bool has_model_context{m_model_context != ""};
42 yarp::os::ResourceFinder resource_finder;
43 if(has_model_context)
44 {
45 resource_finder.setDefaultContext(m_model_context);
46 model_path = resource_finder.findFile(m_model_name);
47 }
48 else
49 {
50 model_path = m_model_name;
51 }
52 init_LLM(model_path);
53 bool has_prompt_file{m_prompt_file != ""};
54 bool has_prompt_context{m_prompt_context != ""};
55 if(has_prompt_context)
56 {
57 resource_finder.setDefaultContext(m_prompt_context);
58 }
59
60 if(has_prompt_file)
61 {
62 std::string prompt_file_fullpath = resource_finder.findFile(m_prompt_file);
63 auto stream = std::ifstream(prompt_file_fullpath);
64 if (!stream)
65 {
66 yCWarning(LLAMA2DEVICE) << "File:" << prompt_file_fullpath << "does not exist or path is invalid";
67 }
68 else
69 {
70 std::ostringstream sstr;
71 sstr << stream.rdbuf(); //Reads the entire file into the stringstream
72 if(!setPrompt(sstr.str()))
73 {
74 return false;
75 }
76 }
77 }
78
79 return true;
80}
81
82// method for the initialization of the LLM model
83yarp::dev::ReturnValue Llama2Device::init_LLM(const std::string &model_path)
84{
85 // start llama-server in background
86 startLlama2Server(model_path, m_context, m_ngl);
87
89}
90
92{
93 nlohmann::json request_json;
94 request_json["model"] = "default";
95 request_json["max_tokens"] = m_npredict;
96
97 // Pass the conversation to the model
98 nlohmann::json messages_array = nlohmann::json::array();
99 for (const auto& msg : m_conversation) {
100 messages_array.push_back({{"role", msg.type}, {"content", msg.content}});
101 }
102
103 // Add new question to the conversation
104 messages_array.push_back({{"role", "user"}, {"content", question}});
105
106 // Insert array messages into the request
107 request_json["messages"] = messages_array;
108
109 // Convert JSON to string and send the request to the model
110 std::string risposta = sendPostRequest(request_json.dump());
111
112 // extract model's answer
113 std::string final_output = extractContent(risposta);
114
115 // add question to the conversation
117 message.type = "user";
118 message.content = question;
119 m_conversation.push_back(message);
120
121 // variable used to store the complete answer generated by the model
122 final_output = extractContent(risposta);
123
124 // add model answer to the conversation
125 message.type = "assitant";
126 message.content = final_output;
127 m_conversation.push_back(message);
128
129 // write model answer inside oAnswer
130 oAnswer.type = "assistant";
131 oAnswer.content = final_output;
132
134}
135
136
138{
139 // check if a prompt is already set
140 if(prompt_set == true){
141 yCError(LLAMA2DEVICE)<< "A prompt is already set. You must delete conversation first";
143 }
144 // if the prompt is not already set, set the prompt
145 prompt_set = true;
146 first_prompt_set = true;
147
148 m_prompt.type = "system";
149 m_prompt.content = prompt;
150
151 // add prompt to the conversation
153 message.type = "system";
154 message.content = prompt;
155 m_conversation.push_back(message);
156
158}
159
160
162{
163 // check if the prompt is set
164 if(prompt_set == false){
165 yCError(LLAMA2DEVICE) << "Prompt is not set";
167 }
168 // if prompt is set, read it
169 oPrompt = m_prompt.content;
170
172}
173
174
175yarp::dev::ReturnValue Llama2Device::getConversation(std::vector<yarp::dev::LLM_Message> &oConversation)
176{
177 // check if the conversation is empty
178 if(m_conversation.empty()){
179 yCInfo(LLAMA2DEVICE) << "The conversation is empty";
181 }
182
183 // if the conversation is not empty, print it
184 for (const auto& msg : oConversation) {
185 yCInfo(LLAMA2DEVICE) << "Role: " << msg.type;
186 yCInfo(LLAMA2DEVICE) << "Content: " << msg.content;
187 yCInfo(LLAMA2DEVICE) << "-------------------";
188 }
189
190 // set the value of the conversation
191 oConversation = m_conversation;
193}
194
195
197{
198 // check if the conversation is empty
199 if (m_conversation.empty()){
200 yCInfo(LLAMA2DEVICE) << "Conversation is already empty";
202 }
203 // if not empty, clear the conversation
204 m_conversation.clear();
205 prompt_set = false;
206 first_prompt_set = false;
207
208 yCInfo(LLAMA2DEVICE) << "Conversation deleted";
210}
211
216
218{
219 yCInfo(LLAMA2DEVICE) << "Closing";
220 return true;
221}
222
224 std::cout << "Llama2Device Methods:" << std::endl;
225 std::cout << "1. setPrompt" << std::endl;
226 std::cout << " - Set the initial prompt for the model." << std::endl;
227 std::cout << "2. ask" << std::endl;
228 std::cout << " - Ask a question to the model and get an answer." << std::endl;
229 std::cout << "3. getConversation" << std::endl;
230 std::cout << " - Prints the current conversation with the model." << std::endl;
231 std::cout << "4. deleteConversation" << std::endl;
232 std::cout << " - Deletes the current conversation" << std::endl;
233 std::cout << "5. readPrompt" << std::endl;
234 std::cout << " - Prints the last prompt provided to the model" << std::endl;
235 std::cout << "6. help" << std::endl;
236 std::cout << " - Prints the list of possible commands and their description." << std::endl;
237}
const yarp::os::LogComponent & LLAMA2DEVICE()
std::string sendPostRequest(const std::string &jsonData)
Definition Utils.cpp:18
void startLlama2Server(const std::string &model_path, int m_context, int m_gpulayers)
Definition Utils.cpp:61
std::string extractContent(const std::string &jsonResponse)
Definition Utils.cpp:3
bool parseParams(const yarp::os::Searchable &config) override
Parse the DeviceDriver parameters.
yarp::dev::ReturnValue refreshConversation() noexcept override
Refresh the conversation.
yarp::dev::ReturnValue ask(const std::string &question, yarp::dev::LLM_Message &oAnswer) override
Performs a question.
bool close() override
Close the DeviceDriver.
yarp::dev::ReturnValue init_LLM(const std::string &model_path)
yarp::dev::ReturnValue readPrompt(std::string &oPrompt) override
Retrieves the provided prompt.
yarp::dev::ReturnValue setPrompt(const std::string &prompt) override
Performs a question.
bool open(yarp::os::Searchable &config) override
Open the DeviceDriver.
yarp::dev::ReturnValue getConversation(std::vector< yarp::dev::LLM_Message > &oConversation) override
Retrieves the whole conversation.
yarp::dev::ReturnValue deleteConversation() noexcept override
Delete the conversation and clear the system context from any internally stored context.
std::string type
type of message (can be prompt, assistant, function, ...)
std::string content
content of the message
@ return_value_error_generic
Method was successfully executed.
@ TraceType
Definition Log.h:92
Helper class for finding config files and other external resources.
bool setDefaultContext(const std::string &contextName)
Sets the context for the current ResourceFinder object.
std::string findFile(const std::string &name)
Find the full path to a file.
A base class for nested structures that can be searched.
Definition Searchable.h:31
#define yCInfo(component,...)
#define yCError(component,...)
#define yCWarning(component,...)
#define YARP_LOG_COMPONENT(name,...)
For streams capable of holding different kinds of content, check what they actually have.