125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424 | @dataclass(init=False)
class GoogleModel(Model):
"""A model that uses Gemini via `generativelanguage.googleapis.com` API.
This is implemented from scratch rather than using a dedicated SDK, good API documentation is
available [here](https://ai.google.dev/api).
Apart from `__init__`, all methods are private or match those of the base class.
"""
client: genai.Client = field(repr=False)
_model_name: GoogleModelName = field(repr=False)
_provider: Provider[genai.Client] = field(repr=False)
_url: str | None = field(repr=False)
_system: str = field(default='google', repr=False)
def __init__(
self,
model_name: GoogleModelName,
*,
provider: Literal['google-gla', 'google-vertex'] | Provider[genai.Client] = 'google-gla',
profile: ModelProfileSpec | None = None,
):
"""Initialize a Gemini model.
Args:
model_name: The name of the model to use.
provider: The provider to use for authentication and API access. Can be either the string
'google-gla' or 'google-vertex' or an instance of `Provider[httpx.AsyncClient]`.
If not provided, a new provider will be created using the other parameters.
profile: The model profile to use. Defaults to a profile picked by the provider based on the model name.
"""
self._model_name = model_name
if isinstance(provider, str):
provider = GoogleProvider(vertexai=provider == 'google-vertex') # pragma: lax no cover
self._provider = provider
self._system = provider.name
self.client = provider.client
self._profile = profile or provider.model_profile
@property
def base_url(self) -> str:
return self._provider.base_url
async def request(
self,
messages: list[ModelMessage],
model_settings: ModelSettings | None,
model_request_parameters: ModelRequestParameters,
) -> ModelResponse:
check_allow_model_requests()
model_settings = cast(GoogleModelSettings, model_settings or {})
response = await self._generate_content(messages, False, model_settings, model_request_parameters)
return self._process_response(response)
@asynccontextmanager
async def request_stream(
self,
messages: list[ModelMessage],
model_settings: ModelSettings | None,
model_request_parameters: ModelRequestParameters,
) -> AsyncIterator[StreamedResponse]:
check_allow_model_requests()
model_settings = cast(GoogleModelSettings, model_settings or {})
response = await self._generate_content(messages, True, model_settings, model_request_parameters)
yield await self._process_streamed_response(response) # type: ignore
@property
def model_name(self) -> GoogleModelName:
"""The model name."""
return self._model_name
@property
def system(self) -> str:
"""The system / model provider."""
return self._system
def _get_tools(self, model_request_parameters: ModelRequestParameters) -> list[ToolDict] | None:
tools: list[ToolDict] = [
ToolDict(function_declarations=[_function_declaration_from_tool(t)])
for t in model_request_parameters.function_tools
]
if model_request_parameters.output_tools:
tools += [
ToolDict(function_declarations=[_function_declaration_from_tool(t)])
for t in model_request_parameters.output_tools
]
return tools or None
def _get_tool_config(
self, model_request_parameters: ModelRequestParameters, tools: list[ToolDict] | None
) -> ToolConfigDict | None:
if not model_request_parameters.allow_text_output and tools:
names: list[str] = []
for tool in tools:
for function_declaration in tool.get('function_declarations') or []:
if name := function_declaration.get('name'): # pragma: no branch
names.append(name)
return _tool_config(names)
else:
return None
@overload
async def _generate_content(
self,
messages: list[ModelMessage],
stream: Literal[False],
model_settings: GoogleModelSettings,
model_request_parameters: ModelRequestParameters,
) -> GenerateContentResponse: ...
@overload
async def _generate_content(
self,
messages: list[ModelMessage],
stream: Literal[True],
model_settings: GoogleModelSettings,
model_request_parameters: ModelRequestParameters,
) -> Awaitable[AsyncIterator[GenerateContentResponse]]: ...
async def _generate_content(
self,
messages: list[ModelMessage],
stream: bool,
model_settings: GoogleModelSettings,
model_request_parameters: ModelRequestParameters,
) -> GenerateContentResponse | Awaitable[AsyncIterator[GenerateContentResponse]]:
tools = self._get_tools(model_request_parameters)
response_mime_type = None
response_schema = None
if model_request_parameters.output_mode == 'native':
if tools:
raise UserError('Gemini does not support structured output and tools at the same time.')
response_mime_type = 'application/json'
output_object = model_request_parameters.output_object
assert output_object is not None
response_schema = self._map_response_schema(output_object)
elif model_request_parameters.output_mode == 'prompted' and not tools:
response_mime_type = 'application/json'
tool_config = self._get_tool_config(model_request_parameters, tools)
system_instruction, contents = await self._map_messages(messages)
http_options: HttpOptionsDict = {
'headers': {'Content-Type': 'application/json', 'User-Agent': get_user_agent()}
}
if timeout := model_settings.get('timeout'):
if isinstance(timeout, (int, float)):
http_options['timeout'] = int(1000 * timeout)
else:
raise UserError('Google does not support setting ModelSettings.timeout to a httpx.Timeout')
config = GenerateContentConfigDict(
http_options=http_options,
system_instruction=system_instruction,
temperature=model_settings.get('temperature'),
top_p=model_settings.get('top_p'),
max_output_tokens=model_settings.get('max_tokens'),
stop_sequences=model_settings.get('stop_sequences'),
presence_penalty=model_settings.get('presence_penalty'),
frequency_penalty=model_settings.get('frequency_penalty'),
safety_settings=model_settings.get('google_safety_settings'),
thinking_config=model_settings.get('google_thinking_config'),
labels=model_settings.get('google_labels'),
tools=cast(ToolListUnionDict, tools),
tool_config=tool_config,
response_mime_type=response_mime_type,
response_schema=response_schema,
)
func = self.client.aio.models.generate_content_stream if stream else self.client.aio.models.generate_content
return await func(model=self._model_name, contents=contents, config=config) # type: ignore
def _process_response(self, response: GenerateContentResponse) -> ModelResponse:
if not response.candidates or len(response.candidates) != 1:
raise UnexpectedModelBehavior('Expected exactly one candidate in Gemini response') # pragma: no cover
candidate = response.candidates[0]
if candidate.content is None or candidate.content.parts is None:
if candidate.finish_reason == 'SAFETY':
raise UnexpectedModelBehavior('Safety settings triggered', str(response))
else:
raise UnexpectedModelBehavior(
'Content field missing from Gemini response', str(response)
) # pragma: no cover
parts = candidate.content.parts or []
vendor_id = response.response_id or None
vendor_details: dict[str, Any] | None = None
finish_reason = candidate.finish_reason
if finish_reason: # pragma: no branch
vendor_details = {'finish_reason': finish_reason.value}
usage = _metadata_as_usage(response)
usage.requests = 1
return _process_response_from_parts(
parts, response.model_version or self._model_name, usage, vendor_id=vendor_id, vendor_details=vendor_details
)
async def _process_streamed_response(self, response: AsyncIterator[GenerateContentResponse]) -> StreamedResponse:
"""Process a streamed response, and prepare a streaming response to return."""
peekable_response = _utils.PeekableAsyncStream(response)
first_chunk = await peekable_response.peek()
if isinstance(first_chunk, _utils.Unset):
raise UnexpectedModelBehavior('Streamed response ended without content or tool calls') # pragma: no cover
return GeminiStreamedResponse(
_model_name=self._model_name,
_response=peekable_response,
_timestamp=first_chunk.create_time or _utils.now_utc(),
)
async def _map_messages(self, messages: list[ModelMessage]) -> tuple[ContentDict | None, list[ContentUnionDict]]:
contents: list[ContentUnionDict] = []
system_parts: list[PartDict] = []
for m in messages:
if isinstance(m, ModelRequest):
message_parts: list[PartDict] = []
for part in m.parts:
if isinstance(part, SystemPromptPart):
system_parts.append({'text': part.content})
elif isinstance(part, UserPromptPart):
message_parts.extend(await self._map_user_prompt(part))
elif isinstance(part, ToolReturnPart):
message_parts.append(
{
'function_response': {
'name': part.tool_name,
'response': part.model_response_object(),
'id': part.tool_call_id,
}
}
)
elif isinstance(part, RetryPromptPart):
if part.tool_name is None:
message_parts.append({'text': part.model_response()}) # pragma: no cover
else:
message_parts.append(
{
'function_response': {
'name': part.tool_name,
'response': {'call_error': part.model_response()},
'id': part.tool_call_id,
}
}
)
else:
assert_never(part)
# Google GenAI requires at least one part in the message.
if not message_parts:
message_parts = [{'text': ''}]
contents.append({'role': 'user', 'parts': message_parts})
elif isinstance(m, ModelResponse):
contents.append(_content_model_response(m))
else:
assert_never(m)
if instructions := self._get_instructions(messages):
system_parts.insert(0, {'text': instructions})
system_instruction = ContentDict(role='user', parts=system_parts) if system_parts else None
return system_instruction, contents
async def _map_user_prompt(self, part: UserPromptPart) -> list[PartDict]:
if isinstance(part.content, str):
return [{'text': part.content}]
else:
content: list[PartDict] = []
for item in part.content:
if isinstance(item, str):
content.append({'text': item})
elif isinstance(item, BinaryContent):
# NOTE: The type from Google GenAI is incorrect, it should be `str`, not `bytes`.
base64_encoded = base64.b64encode(item.data).decode('utf-8')
content.append({'inline_data': {'data': base64_encoded, 'mime_type': item.media_type}}) # type: ignore
elif isinstance(item, VideoUrl) and item.is_youtube:
content.append({'file_data': {'file_uri': item.url, 'mime_type': item.media_type}})
elif isinstance(item, FileUrl):
if self.system == 'google-gla' or item.force_download:
downloaded_item = await download_item(item, data_format='base64')
inline_data = {'data': downloaded_item['data'], 'mime_type': downloaded_item['data_type']}
content.append({'inline_data': inline_data}) # type: ignore
else:
content.append({'file_data': {'file_uri': item.url, 'mime_type': item.media_type}})
else:
assert_never(item)
return content
def _map_response_schema(self, o: OutputObjectDefinition) -> dict[str, Any]:
response_schema = o.json_schema.copy()
if o.name:
response_schema['title'] = o.name
if o.description:
response_schema['description'] = o.description
return response_schema
|