Merge pull request #69 from GoogleCloudPlatform/gemini-15

Gemini 1.5 stuff

docs/build/docs/processors.md

@@ -895,6 +895,9 @@ Vertex AI processor.
     * **returnErrors** (*bool**, **optional*) – Set to true to return errors
 
 
+    * **apiVersion** (*str**, **optional*) – API version, defaults to “v1”.
+
+
     * **request** (*dict*) – Request.
 
 

examples/gemini-pro-slack-bot/slack-bot.yaml

@@ -104,11 +104,13 @@ pipeline:
       appId: "{{ slackAppId }}"
       multimodal: true
       messages: slack_thread
+      noQuestionPrompt: |
+        Answer the question in this audio clip or image.
+        
 %{ if api_enabled ~}
       prompt: |
-        Do not call functions unless my question is strictly related to Alphabet financials which
-        you don't know the answer for. Answer questions without calling functions if you know the
-        answer well enough.
+        Do not call functions unless my question is strictly related to current and historical
+        Alphabet financial information.
         
 %{ endif ~}
 
@@ -125,10 +127,10 @@ pipeline:
       {% if slack_should_reply %}1{% endif %}
     output: vertex_response
     config:
-      apiVersion: "v1beta"
+      apiVersion: "v1beta1"
       location: "${vertex_region}"
       modelId: "${vertex_model}"
-      method: "streamGenerateContent"
+      method: "generateContent"
       returnErrors: true
 %{ if api_enabled ~}
       callFunctions:
@@ -149,8 +151,8 @@ pipeline:
           functionDeclarations:
             - name: search_vertex
               description: |
-                This function is only for getting current information on Alphabet
-                financials, do not use it for anything else. 
+                This function is only for getting up-to-date information on Alphabet
+                financials.
               parameters:
                 type: object
                 properties:
@@ -159,6 +161,9 @@ pipeline:
                     description: Search query
                 required:
                   - query
+        # toolConfig:
+        #  functionCallingConfig:
+        #    mode: AUTO
 %{ endif ~}
         safetySettings:
           - category: HARM_CATEGORY_SEXUALLY_EXPLICIT

examples/gemini-pro-slack-bot/variables.tf

@@ -37,13 +37,13 @@ variable "vertex_region" {
 variable "vertex_model" {
   description = "Vertex AI model to use"
   type        = string
-  default     = "gemini-1.0-pro"
+  default     = "gemini-1.5-pro-preview-0409"
 }
 
 variable "vertex_model_multimodal" {
   description = "Vertex AI multimodal model to use"
   type        = string
-  default     = "gemini-1.0-pro-vision"
+  default     = "gemini-1.5-pro-preview-0409"
 }
 
 variable "slack_token" {

examples/gemini-pro-slack-bot/vertex-search.yaml

@@ -53,20 +53,38 @@ pipeline:
               version: "stable"
           extractiveContentSpec:
             maxExtractiveAnswerCount: 1
+            maxExtractiveSegmentCount: 1
 
   # Log the response
   - type: output.logger
     config:
       message: "Vertex Search response: {{ vertex_response|json_encode }}"
 
+  - type: processor.setvariable
+    description: Process Vertex Search output
+    output: response
+    config:
+      fromJson: true
+      value: |
+        {%- set resp = [] -%}
+        {% for result in vertex_response.results %}
+          {% if "derivedStructData" in result.document and "extractive_answers" in result.document.derivedStructData %}
+            {% for answer in result.document.derivedStructData.extractive_answers %}
+              {% set _ = resp.append({"link": result.document.derivedStructData.link, "contents": answer.content, "pageNumber": answer.pageNumber }) %}
+            {% endfor %}    
+          {% endif %}
+        {%- endfor -%}
+        {{ resp|json_encode }}
+
+  - type: output.logger
+    config:
+      message: "Vertex Search processed response: {{ response|json_encode }}"
+
   - type: output.httpresponse
     runIf: "{% if using_webserver %}1{% endif %}"
     config:
       statusCode: 200
       headers:
         content-type: "application/json"
       body: |
-        {%- set resp = [] -%}
-        {%- for result in vertex_response.results -%}
-
-        {%- endfor -%}
+        {{ {"response":response}|json_encode }}

processors/slack.py

@@ -73,7 +73,7 @@ def download_slack(self, url, token):
         response.raise_for_status()
         return base64.b64encode(response.content).decode('ascii')
 
-    def _slack_message_to_parts(self, message, token, multi_modal):
+    def _slack_message_to_parts(self, message, token, multi_modal, no_question):
         parts = []
         if 'files' in message and multi_modal:
             for file in message['files']:
@@ -84,6 +84,10 @@ def _slack_message_to_parts(self, message, token, multi_modal):
                         extra={'file': file})
                     continue
                 if file['mimetype'][0:6] == 'image/':
+                    self.logger.info(
+                        'Downloaded Slack image (720p version): %s (mime %s)' %
+                        (file['thumb_720']),
+                        extra={'mimetype': file['mimetype']})
                     if 'thumb_720' in file:
                         parts.append({
                             'inlineData': {
@@ -95,6 +99,10 @@ def _slack_message_to_parts(self, message, token, multi_modal):
                             }
                         })
                     elif 'url_private_download' in file:
+                        self.logger.info(
+                            'Downloaded Slack image (full version): %s (mime %s)'
+                            % (file['url_private_download'], file['mimetype']),
+                            extra={'mimetype': file['mimetype']})
                         parts.append({
                             'inlineData': {
                                 'mimeType':
@@ -105,6 +113,10 @@ def _slack_message_to_parts(self, message, token, multi_modal):
                             }
                         })
                 elif 'url_private_download' in file:
+                    self.logger.info(
+                        'Downloaded Slack file: %s (mime %s)' %
+                        (file['url_private_download'], file['mimetype']),
+                        extra={'mimetype': file['mimetype']})
                     parts.append({
                         'inlineData': {
                             'mimeType':
@@ -116,6 +128,9 @@ def _slack_message_to_parts(self, message, token, multi_modal):
                     })
         if 'text' in message and message['text'] != '':
             parts.append({'text': message['text']})
+        elif no_question != '':
+            parts.append({'text': no_question})
+
         return parts
 
     def process(self, output_var='slack'):
@@ -136,6 +151,9 @@ def process(self, output_var='slack'):
         request_params = self._jinja_expand_dict_all(self.config['request'],
                                                      'request')
 
+        mode = self._jinja_expand_string(
+            self.config['mode'], 'mode') if 'mode' in self.config else 'api'
+
         if mode == 'processMessages' or mode == 'lastImage':
             if 'messages' not in self.config:
                 raise NotConfiguredException('No Slack messages specified.')
@@ -148,6 +166,10 @@ def process(self, output_var='slack'):
             messages = self._jinja_expand_expr(self.config['messages'],
                                                'messages')
 
+            no_question = self._jinja_expand_string(
+                self.config['noQuestionPrompt'], 'no_question_prompt'
+            ) if 'noQuestionPrompt' in self.config else "Answer the question in this audio clip or image."
+
             processed = []
             if 'messages' in messages:
                 messages = messages['messages']
@@ -157,17 +179,19 @@ def process(self, output_var='slack'):
                 if 'app_id' in message:
                     if message['app_id'] == app_id:
                         parts = self._slack_message_to_parts(
-                            message, token, multi_modal)
+                            message, token, multi_modal, no_question)
                         if len(parts) > 0:
                             new_message = {'role': 'MODEL', 'parts': parts}
                 else:
                     parts = self._slack_message_to_parts(
-                        message, token, multi_modal)
+                        message, token, multi_modal, no_question)
                     if len(parts) > 0:
                         new_message = {'role': 'USER', 'parts': parts}
                 if new_message:
                     processed.append(new_message)
 
+            print('PROCESSED', processed)
+
             # Prepend an initial prompt that can be instructions or such
             if 'prompt' in self.config:
                 prompt_added = False

processors/vertexai.py

@@ -71,20 +71,21 @@ def process(self, output_var='vertexai'):
             self.config['servingConfig'], 'serving_config'
         ) if 'servingConfig' in self.config else 'default_config'
 
-        api_version = self._jinja_expand_string(self.config['apiVersion']) if 'apiVersion' in self.config else 'v1'
+        api_version = self._jinja_expand_string(
+            self.config['apiVersion']) if 'apiVersion' in self.config else 'v1'
 
         method = self._jinja_expand_string(
             self.config['method'],
             'method') if 'method' in self.config else 'search'
         if mode == 'search':
             if engine_id:
                 api_path = 'https://%s-discoveryengine.googleapis.com/%s/projects/%s/locations/%s/collections/%s/engineId/%s/servingConfigs/%s:%s' % (
-                    api_version, location, project, location, collection, engine_id,
-                    serving_config, method)
+                    api_version, location, project, location, collection,
+                    engine_id, serving_config, method)
             else:
                 api_path = 'https://%s-discoveryengine.googleapis.com/%s/projects/%s/locations/%s/collections/%s/dataStores/%s/servingConfigs/%s:%s' % (
-                    location, api_version, project, location, collection, datastore_id,
-                    serving_config, method)
+                    location, api_version, project, location, collection,
+                    datastore_id, serving_config, method)
 
             return_errors = False
             if 'returnErrors' in self.config:

processors/vertexgenai.py

@@ -180,15 +180,37 @@ def process(self, output_var='vertexgenai'):
         try:
             response.raise_for_status()
         except Exception as e:
-            if return_errors:
-                response_json = response.json()
-                for err in response_json:
-                    if 'error' in err and 'message' in err['error']:
-                        return {output_var: {'error': err['error']['message']}}
             self.logger.error('Error calling %s: %s' % (e.request.url, e),
                               extra={'response': e.response.text})
+            if return_errors:
+                try:
+                    response_json = response.json()
+                    for err in response_json:
+                        if isinstance(
+                                err, dict
+                        ) and 'error' in err and 'message' in err['error']:
+                            return {
+                                output_var: {
+                                    'error': err['error']['message']
+                                }
+                            }
+                        else:
+                            return {output_var: {'error': err}}
+                except Exception as _:
+                    return {output_var: {'error': e.response.text}}
+            raise e
+        try:
+            response_json = response.json()
+        except Exception as e:
+            self.logger.error('Response was not JSON from %s: %s' %
+                              (e.request.url, e),
+                              extra={'response': e.response.text})
+            if return_errors:
+                return {output_var: {'error': e.response.text}}
             raise e
-        response_json = response.json()
+
+        if isinstance(response_json, dict):
+            response_json = [response_json]
 
         # Check if functions need to be called
         if 'callFunctions' in self.config:
@@ -283,6 +305,9 @@ def process(self, output_var='vertexgenai'):
                     raise e
                 response_json = response.json()
 
+                if isinstance(response_json, dict):
+                    response_json = [response_json]
+
         return {
             output_var: response_json,
         }