diff --git a/pkg/component/application/github/v0/README.mdx b/pkg/component/application/github/v0/README.mdx index 3f409ac43..2dfa31769 100644 --- a/pkg/component/application/github/v0/README.mdx +++ b/pkg/component/application/github/v0/README.mdx @@ -323,7 +323,7 @@ Get the review comments in a pull request. The comments can be on a specific lin | Field | Field ID | Type | Note | | :--- | :--- | :--- | :--- | | Comment body | `body` | string | Body of the comment. | -| Commit SHA | `commitId` | string | SHA of the commit on which you want to comment. | +| Commit SHA | `commit-id` | string | SHA of the commit on which you want to comment. | | Comment created at | `created-at` | string | Time the comment was created. | | Comment id | `id` | integer | ID of the comment. | | In Reply To | `in-reply-to-id` | integer | ID of the comment this comment is in reply to. | @@ -395,7 +395,7 @@ The comment to be added. | :--- | :--- | :--- | :--- | | Comment ID (optional) | `id` | integer | ID of the comment. | | In Reply To (optional) | `in-reply-to-id` | integer | ID of the comment this comment is in reply to. | -| Commit SHA (optional) | `commitId` | string | SHA of the commit on which you want to comment. | +| Commit SHA (optional) | `commit-id` | string | SHA of the commit on which you want to comment. | | Comment Body (optional) | `body` | string | Body of the comment. | | Comment Path (optional) | `path` | string | Path of the file the comment is on. | | Comment End Line (optional) | `line` | integer | The line of the blob in the pull request diff that the comment applies to. For a multi-line comment, the last line of the range that your comment applies to. | diff --git a/pkg/component/data/googledrive/v0/config/setup.json b/pkg/component/data/googledrive/v0/config/setup.json index cc59e05ca..f04853cef 100644 --- a/pkg/component/data/googledrive/v0/config/setup.json +++ b/pkg/component/data/googledrive/v0/config/setup.json @@ -24,6 +24,7 @@ "accessUrl": "https://oauth2.googleapis.com/token", "scopes": [ "https://www.googleapis.com/auth/drive.readonly", + "https://www.googleapis.com/auth/drive.file", "https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/userinfo.profile" ] diff --git a/pkg/component/data/googlesheets/v0/README.mdx b/pkg/component/data/googlesheets/v0/README.mdx new file mode 100644 index 000000000..6a5014d43 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/README.mdx @@ -0,0 +1,514 @@ +--- +title: "Google Sheets" +lang: "en-US" +draft: false +description: "Learn about how to set up a VDP Google Sheets component https://github.com/instill-ai/instill-core" +--- + +The Google Sheets component is a data component that allows users to connect to and interact with Google Sheets spreadsheets to read and write data. +It can carry out the following tasks: +- [Create Spreadsheet](#create-spreadsheet) +- [Delete Spreadsheet](#delete-spreadsheet) +- [Add Sheet](#add-sheet) +- [Delete Sheet](#delete-sheet) +- [Create Spreadsheet Column](#create-spreadsheet-column) +- [Delete Spreadsheet Column](#delete-spreadsheet-column) +- [List Rows](#list-rows) +- [Lookup Rows](#lookup-rows) +- [Get Row](#get-row) +- [Get Multiple Rows](#get-multiple-rows) +- [Insert Row](#insert-row) +- [Insert Multiple Rows](#insert-multiple-rows) +- [Update Row](#update-row) +- [Update Multiple Rows](#update-multiple-rows) +- [Delete Row](#delete-row) +- [Delete Multiple Rows](#delete-multiple-rows) + + + +## Release Stage + +`Alpha` + + + +## Configuration + +The component definition and tasks are defined in the [definition.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/googlesheets/v0/config/definition.json) and [tasks.json](https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/googlesheets/v0/config/tasks.json) files respectively. + + + + +## Setup + + +In order to communicate with Google, the following connection details need to be +provided. You may specify them directly in a pipeline recipe as key-value pairs +within the component's `setup` block, or you can create a **Connection** from +the [**Integration Settings**](https://www.instill.tech/docs/vdp/integration) +page and reference the whole `setup` as `setup: +${connection.}`. + +
+ +| Field | Field ID | Type | Note | +| :--- | :--- | :--- | :--- | +| Refresh Token | `refresh-token` | string | Refresh token for the Google Sheets API. For more information about how to create tokens, please refer to the Google Sheets API documentation and OAuth 2.0 documentation. | + +
+ + + + +## Supported Tasks + +### Create Spreadsheet + +Create a new Google Sheets spreadsheet with multiple sheets. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_CREATE_SPREADSHEET` | +| Title | `title` | string | Title of the new spreadsheet. | +| [Sheets](#create-spreadsheet-sheets) (required) | `sheets` | array[object] | Configuration for sheets to create. | +
+ + +
+ Input Objects in Create Spreadsheet + +

Sheets

+ +Configuration for sheets to create. + +
+ +| Field | Field ID | Type | Note | +| :--- | :--- | :--- | :--- | +| Headers | `headers` | array | Column headers for the sheet. | +| Sheet Name | `name` | string | Name of the sheet. | +
+
+ + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Spreadsheet ID | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +
+ + +### Delete Spreadsheet + +Delete a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_DELETE_SPREADSHEET` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Success | `success` | string | Result of the operation. | +
+ + +### Add Sheet + +Add a new sheet to an existing Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_ADD_SHEET` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Headers | `headers` | array[string] | Column headers for the sheet. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Success | `result` | string | Result of the operation. | +
+ + +### Delete Sheet + +Remove a sheet from a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_DELETE_SHEET` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Success | `result` | string | Result of the operation. | +
+ + +### Create Spreadsheet Column + +Add a new column to a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_CREATE_SPREADSHEET_COLUMN` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Column Name (required) | `column-name` | string | Name of the column. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Success | `result` | string | Result of the operation. | +
+ + +### Delete Spreadsheet Column + +Delete a column from a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_DELETE_SPREADSHEET_COLUMN` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Column Name (required) | `column-name` | string | Name of the column. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Success | `result` | string | Result of the operation. | +
+ + +### List Rows + +List all rows in a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_LIST_ROWS` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Start Row (required) | `start-row` | integer | The starting row number to retrieve (1-based index). | +| End Row (required) | `end-row` | integer | The ending row number to retrieve (1-based index). | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| [Rows Data](#list-rows-rows-data) | `rows` | array[object] | Rows data in JSON format where keys are column names. | +
+ + +### Lookup Rows + +Find multiple rows based on column value in a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_LOOKUP_ROWS` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Column Name (required) | `column-name` | string | Name of the column. | +| Search Value (required) | `value` | string | Value to search for in the specified column. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Row Numbers (optional) | `row-numbers` | array[integer] | Row numbers to update (1-based indices). | +| [Rows Data](#lookup-rows-rows-data) | `rows` | array[object] | Rows data in JSON format where keys are column names. | +
+ + +### Get Row + +Get a single row from a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_GET_ROW` | +| Row Number (required) | `row-number` | integer | The row number to retrieve (1-based index). | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Row Data | `row` | object | Row data in JSON format where keys are column names. | +
+ + +### Get Multiple Rows + +Get multiple rows from a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_GET_MULTIPLE_ROWS` | +| Row Numbers (required) | `row-numbers` | array[integer] | The row numbers to retrieve (1-based indices). | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| [Rows Data](#get-multiple-rows-rows-data) | `rows` | array[object] | Rows data in JSON format where keys are column names. | +
+ + +### Insert Row + +Insert a single row into a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_INSERT_ROW` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Row Data (required) | `row` | object | Row data in JSON format where keys are column names. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Row Number | `row-number` | integer | Row number to update (1-based index). | +| Row Data | `row` | object | Row data in JSON format where keys are column names. | +
+ + +### Insert Multiple Rows + +Insert multiple rows into a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_INSERT_MULTIPLE_ROWS` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| [Rows Data](#insert-multiple-rows-rows-data) (required) | `rows` | array[object] | Rows data in JSON format where keys are column names. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Row Numbers | `row-numbers` | array[integer] | Row numbers to update (1-based indices). | +| [Rows Data](#insert-multiple-rows-rows-data) | `rows` | array[object] | Rows data in JSON format where keys are column names. | +
+ + +### Update Row + +Update a row in a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_UPDATE_ROW` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Row Number (required) | `row-number` | integer | Row number to update (1-based index). | +| Row Data (required) | `row` | object | Row data in JSON format where keys are column names. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Row Data | `row` | object | Row data in JSON format where keys are column names. | +
+ + +### Update Multiple Rows + +Update multiple rows in a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_UPDATE_MULTIPLE_ROWS` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Row Numbers (required) | `row-numbers` | array[integer] | Row numbers to update (1-based indices). | +| [Rows Data](#update-multiple-rows-rows-data) (required) | `rows` | array[object] | Rows data in JSON format where keys are column names. | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| [Rows Data](#update-multiple-rows-rows-data) | `rows` | array[object] | Rows data in JSON format where keys are column names. | +
+ + +### Delete Row + +Delete a row from a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_DELETE_ROW` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Row Number (required) | `row-number` | integer | Row number to update (1-based index). | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Success | `result` | string | Result of the operation. | +
+ + +### Delete Multiple Rows + +Delete multiple rows from a Google Sheets spreadsheet. + +
+ +| Input | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Task ID (required) | `task` | string | `TASK_DELETE_MULTIPLE_ROWS` | +| Spreadsheet ID (required) | `shared-link` | string | Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'. | +| Sheet Name (required) | `sheet-name` | string | Name of the sheet. | +| Row Numbers (required) | `row-numbers` | array[integer] | Row numbers to update (1-based indices). | +
+ + + + + + +
+ +| Output | ID | Type | Description | +| :--- | :--- | :--- | :--- | +| Success | `result` | string | Result of the operation. | +
+ + + diff --git a/pkg/component/data/googlesheets/v0/assets/google-sheets.svg b/pkg/component/data/googlesheets/v0/assets/google-sheets.svg new file mode 100644 index 000000000..e8bd91971 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/assets/google-sheets.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/pkg/component/data/googlesheets/v0/config/definition.json b/pkg/component/data/googlesheets/v0/config/definition.json new file mode 100644 index 000000000..297f40141 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/config/definition.json @@ -0,0 +1,35 @@ +{ + "availableTasks": [ + "TASK_CREATE_SPREADSHEET", + "TASK_DELETE_SPREADSHEET", + "TASK_ADD_SHEET", + "TASK_DELETE_SHEET", + "TASK_CREATE_SPREADSHEET_COLUMN", + "TASK_DELETE_SPREADSHEET_COLUMN", + "TASK_LIST_ROWS", + "TASK_LOOKUP_ROWS", + "TASK_GET_ROW", + "TASK_GET_MULTIPLE_ROWS", + "TASK_INSERT_ROW", + "TASK_INSERT_MULTIPLE_ROWS", + "TASK_UPDATE_ROW", + "TASK_UPDATE_MULTIPLE_ROWS", + "TASK_DELETE_ROW", + "TASK_DELETE_MULTIPLE_ROWS" + ], + "custom": false, + "documentationUrl": "https://www.instill.tech/docs/component/data/google-sheets", + "icon": "assets/google-sheets.svg", + "id": "google-sheets", + "public": true, + "title": "Google Sheets", + "description": "Connect to and interact with Google Sheets spreadsheets to read and write data.", + "tombstone": false, + "type": "COMPONENT_TYPE_DATA", + "uid": "c4f8e3d2-a1b9-4c7e-9f5d-6b2e8a7d3c1f", + "vendor": "Google", + "vendorAttributes": {}, + "version": "0.1.0", + "sourceUrl": "https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/googlesheets/v0", + "releaseStage": "RELEASE_STAGE_ALPHA" +} diff --git a/pkg/component/data/googlesheets/v0/config/setup.json b/pkg/component/data/googlesheets/v0/config/setup.json new file mode 100644 index 000000000..d91f47285 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/config/setup.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "refresh-token": { + "description": "Refresh token for the Google Sheets API. For more information about how to create tokens, please refer to the Google Sheets API documentation and OAuth 2.0 documentation.", + "instillUpstreamTypes": [ + "reference" + ], + "instillAcceptFormats": [ + "string" + ], + "instillSecret": true, + "instillUIOrder": 1, + "title": "Refresh Token", + "type": "string" + } + }, + "required": [], + "instillEditOnNodeFields": [ + "refresh-token" + ], + "instillOAuthConfig": { + "authUrl": "https://accounts.google.com/o/oauth2/auth", + "accessUrl": "https://oauth2.googleapis.com/token", + "scopes": [ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/userinfo.profile" + ] + }, + "title": "Google Drive Connection", + "type": "object" +} diff --git a/pkg/component/data/googlesheets/v0/config/tasks.json b/pkg/component/data/googlesheets/v0/config/tasks.json new file mode 100644 index 000000000..da06208ae --- /dev/null +++ b/pkg/component/data/googlesheets/v0/config/tasks.json @@ -0,0 +1,725 @@ +{ + "$defs": { + "shared-link": { + "description": "Shared link of the spreadsheet. You can get the shared link by clicking 'Share' button and selecting 'Copy link'.", + "instillFormat": "string", + "instillUIOrder": 0, + "title": "Spreadsheet ID", + "type": "string" + }, + "headers": { + "description": "Column headers for the sheet.", + "type": "array", + "items": { + "type": "string" + }, + "instillFormat": "array:string", + "instillUIOrder": 1, + "title": "Headers" + }, + "sheet-name": { + "description": "Name of the sheet.", + "instillFormat": "string", + "instillUIOrder": 2, + "title": "Sheet Name", + "type": "string" + }, + "column-name": { + "description": "Name of the column.", + "instillFormat": "string", + "instillUIOrder": 3, + "title": "Column Name", + "type": "string" + }, + "row-number": { + "description": "Row number to update (1-based index).", + "instillFormat": "number", + "instillUIOrder": 4, + "title": "Row Number", + "type": "integer" + }, + "row-numbers": { + "description": "Row numbers to update (1-based indices).", + "items": { + "type": "integer" + }, + "instillFormat": "array:number", + "instillUIOrder": 5, + "title": "Row Numbers", + "type": "array" + }, + "row-value": { + "type": "object", + "description": "Row data in JSON format where keys are column names and values are the corresponding cell values", + "title": "Row Data", + "instillFormat": "json", + "instillUIOrder": 6, + "additionalProperties": { + "type": "string", + "description": "Cell value for the corresponding column" + }, + "required": [] + }, + "row": { + "description": "Row data with row number and data.", + "instillFormat": "json", + "instillUIOrder": 7, + "title": "Row Data", + "type": "object", + "properties": { + "row-number": { + "type": "integer", + "instillFormat": "number", + "description": "Row number to update (1-based index)", + "title": "Row Number", + "instillUIOrder": 0 + }, + "row-value": { + "$ref": "#/$defs/row-value", + "instillUIOrder": 1 + } + }, + "required": ["row-number", "data"] + }, + "rows": { + "description": "Multiple rows data with row numbers and data.", + "instillFormat": "array:json", + "items": { + "$ref": "#/$defs/row" + }, + "instillUIOrder": 8, + "title": "Rows Data", + "type": "array" + }, + "success": { + "description": "Result of the operation.", + "instillFormat": "boolean", + "instillUIOrder": 9, + "title": "Success", + "type": "boolean" + } + }, + "TASK_CREATE_SPREADSHEET": { + "instillShortDescription": "Create a new Google Sheets spreadsheet with multiple sheets.", + "input": { + "description": "Please provide the name and sheets configuration for the new spreadsheet.", + "properties": { + "title": { + "description": "Title of the new spreadsheet.", + "instillFormat": "string", + "instillUIOrder": 0, + "title": "Title", + "type": "string" + }, + "sheets": { + "title": "Sheets", + "description": "Configuration for sheets to create.", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "$ref": "#/$defs/sheet-name" + }, + "headers": { + "$ref": "#/$defs/headers" + } + }, + "required": ["name"] + }, + "instillUIOrder": 1 + } + }, + "required": [ + "name", + "sheets" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + } + }, + "required": [ + "shared-link" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_DELETE_SPREADSHEET": { + "instillShortDescription": "Delete a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the shared link of the spreadsheet to delete.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + } + }, + "required": [ + "shared-link" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "success": { + "$ref": "#/$defs/success" + } + }, + "required": [ + "success" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_ADD_SHEET": { + "instillShortDescription": "Add a new sheet to an existing Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and new sheet configuration.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "headers": { + "$ref": "#/$defs/headers" + } + }, + "required": [ + "shared-link", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "success": { + "$ref": "#/$defs/success" + } + }, + "required": [ + "success" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_DELETE_SHEET": { + "instillShortDescription": "Remove a sheet from a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and sheet to remove.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + } + }, + "required": [ + "shared-link", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "success": { + "$ref": "#/$defs/success" + } + }, + "required": [ + "success" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_CREATE_SPREADSHEET_COLUMN": { + "instillShortDescription": "Add a new column to a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and new column information.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "column-name": { + "$ref": "#/$defs/column-name" + } + }, + "required": [ + "shared-link", + "column-name", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "success": { + "$ref": "#/$defs/success" + } + }, + "required": [ + "success" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_DELETE_SPREADSHEET_COLUMN": { + "instillShortDescription": "Delete a column from a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and column to delete.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "column-name": { + "$ref": "#/$defs/column-name" + } + }, + "required": [ + "shared-link", + "column-name", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "success": { + "$ref": "#/$defs/success" + } + }, + "required": [ + "success" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_GET_ROW": { + "instillShortDescription": "Get a single row from a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and row number.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "row-number": { + "description": "The row number to retrieve (1-based index).", + "instillFormat": "number", + "minimum": 1, + "title": "Row Number", + "type": "integer", + "instillUIOrder": 0 + } + }, + "required": [ + "shared-link", + "row-number", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "row": { + "$ref": "#/$defs/row" + } + }, + "required": [ + "row" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_GET_MULTIPLE_ROWS": { + "instillShortDescription": "Get multiple rows from a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and row numbers.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "row-numbers": { + "description": "The row numbers to retrieve (1-based indices).", + "items": { + "minimum": 1, + "type": "integer" + }, + "instillFormat": "array:number", + "title": "Row Numbers", + "type": "array", + "instillUIOrder": 0 + } + }, + "required": [ + "shared-link", + "row-numbers", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "rows": { + "$ref": "#/$defs/rows" + } + }, + "required": [ + "rows" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_LIST_ROWS": { + "instillShortDescription": "List all rows in a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details to list all rows.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "start-row": { + "description": "The starting row number to retrieve (1-based index).", + "instillFormat": "number", + "minimum": 1, + "title": "Start Row", + "type": "integer", + "instillUIOrder": 0 + }, + "end-row": { + "description": "The ending row number to retrieve (1-based index).", + "instillFormat": "number", + "minimum": 1, + "title": "End Row", + "type": "integer", + "instillUIOrder": 1 + } + }, + "required": [ + "shared-link", + "sheet-name", + "start-row", + "end-row" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "rows": { + "$ref": "#/$defs/rows" + } + }, + "required": [ + "rows" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_LOOKUP_ROWS": { + "instillShortDescription": "Find multiple rows based on column value in a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and lookup criteria.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "column-name": { + "$ref": "#/$defs/column-name" + }, + "value": { + "description": "Value to search for in the specified column.", + "instillFormat": "string", + "title": "Search Value", + "type": "string", + "instillUIOrder": 10 + } + }, + "required": [ + "shared-link", + "column-name", + "value", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "rows": { + "$ref": "#/$defs/rows" + } + }, + "required": [ + "rows" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_INSERT_ROW": { + "instillShortDescription": "Insert a single row into a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and row data to insert.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "row-value": { + "$ref": "#/$defs/row-value" + } + }, + "required": [ + "shared-link", + "row", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "row": { + "$ref": "#/$defs/row" + } + }, + "required": [ + "row", + "row-number" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_INSERT_MULTIPLE_ROWS": { + "instillShortDescription": "Insert multiple rows into a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and rows data to insert.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "row-values": { + "type": "array", + "description": "Array of row data in JSON format where keys are column names and values are the corresponding cell values", + "items": { + "$ref": "#/$defs/row-value" + }, + "instillFormat": "array:json", + "instillUIOrder": 2, + "title": "Row Values" + } + }, + "required": [ + "shared-link", + "rows", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "rows": { + "$ref": "#/$defs/rows" + } + }, + "required": [ + "rows" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_UPDATE_ROW": { + "instillShortDescription": "Update a row in a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and row data to update.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "row": { + "$ref": "#/$defs/row" + } + }, + "required": [ + "shared-link", + "row-number", + "row", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "row": { + "$ref": "#/$defs/row" + } + }, + "required": [ + "row" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_UPDATE_MULTIPLE_ROWS": { + "instillShortDescription": "Update multiple rows in a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and rows data to update.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "rows": { + "$ref": "#/$defs/rows" + } + }, + "required": [ + "shared-link", + "row-numbers", + "rows", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "rows": { + "$ref": "#/$defs/rows" + } + }, + "required": [ + "rows" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_DELETE_ROW": { + "instillShortDescription": "Delete a row from a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and Row number to delete.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "row-number": { + "$ref": "#/$defs/row-number" + } + }, + "required": [ + "shared-link", + "row-number", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "success": { + "$ref": "#/$defs/success" + } + }, + "required": [ + "success" + ], + "title": "Output", + "type": "object" + } + }, + "TASK_DELETE_MULTIPLE_ROWS": { + "instillShortDescription": "Delete multiple rows from a Google Sheets spreadsheet.", + "input": { + "description": "Please provide the spreadsheet details and Row numbers to delete.", + "properties": { + "shared-link": { + "$ref": "#/$defs/shared-link" + }, + "sheet-name": { + "$ref": "#/$defs/sheet-name" + }, + "row-numbers": { + "$ref": "#/$defs/row-numbers" + } + }, + "required": [ + "shared-link", + "row-numbers", + "sheet-name" + ], + "title": "Input", + "type": "object" + }, + "output": { + "properties": { + "success": { + "$ref": "#/$defs/success" + } + }, + "required": [ + "success" + ], + "title": "Output", + "type": "object" + } + } +} diff --git a/pkg/component/data/googlesheets/v0/io.go b/pkg/component/data/googlesheets/v0/io.go new file mode 100644 index 000000000..49538489e --- /dev/null +++ b/pkg/component/data/googlesheets/v0/io.go @@ -0,0 +1,206 @@ +package googlesheets + +import ( + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +// taskCreateSpreadsheetInput represents input for creating a new spreadsheet +type taskCreateSpreadsheetInput struct { + Title string `instill:"title"` + Sheets []sheet `instill:"sheets"` +} + +type sheet struct { + Name string `instill:"name"` + Headers []string `instill:"headers"` +} + +// taskCreateSpreadsheetOutput represents output after creating a spreadsheet +type taskCreateSpreadsheetOutput struct { + SharedLink string `instill:"shared-link"` +} + +// taskDeleteSpreadsheetInput represents input for deleting a spreadsheet +type taskDeleteSpreadsheetInput struct { + SharedLink string `instill:"shared-link"` +} + +// taskDeleteSpreadsheetOutput represents output after deleting a spreadsheet +type taskDeleteSpreadsheetOutput struct { + Success bool `instill:"success"` +} + +// taskAddSheetInput represents input for adding a new sheet +type taskAddSheetInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + Headers []string `instill:"headers"` +} + +// taskAddSheetOutput represents output after adding a sheet +type taskAddSheetOutput struct { + Success bool `instill:"success"` +} + +// taskDeleteSheetInput represents input for deleting a sheet +type taskDeleteSheetInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` +} + +// taskDeleteSheetOutput represents output after deleting a sheet +type taskDeleteSheetOutput struct { + Success bool `instill:"success"` +} + +// taskCreateSpreadsheetColumnInput represents input for creating a column +type taskCreateSpreadsheetColumnInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + ColumnName string `instill:"column-name"` +} + +// taskCreateSpreadsheetColumnOutput represents output after creating a column +type taskCreateSpreadsheetColumnOutput struct { + Success bool `instill:"success"` +} + +// taskDeleteSpreadsheetColumnInput represents input for deleting a column +type taskDeleteSpreadsheetColumnInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + ColumnName string `instill:"column-name"` +} + +// taskDeleteSpreadsheetColumnOutput represents output after deleting a column +type taskDeleteSpreadsheetColumnOutput struct { + Success bool `instill:"success"` +} + +// taskListRowsInput represents input for listing rows +type taskListRowsInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + StartRow int `instill:"start-row"` + EndRow int `instill:"end-row"` +} + +// taskListRowsOutput represents output after listing rows +type taskListRowsOutput struct { + Rows []Row `instill:"rows"` +} + +// taskLookupRowsInput represents input for looking up multiple rows +type taskLookupRowsInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + ColumnName string `instill:"column-name"` + Value string `instill:"value"` +} + +// taskLookupRowsOutput represents output after looking up multiple rows +type taskLookupRowsOutput struct { + Rows []Row `instill:"rows"` +} + +// taskGetRowInput represents input for getting a row +type taskGetRowInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + RowNumber int `instill:"row-number"` +} + +// taskGetRowOutput represents output after getting a row +type taskGetRowOutput struct { + Row Row `instill:"row"` +} + +// taskGetMultipleRowsInput represents input for getting multiple rows +type taskGetMultipleRowsInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + RowNumbers []int `instill:"row-numbers"` +} + +// taskGetMultipleRowsOutput represents output after getting multiple rows +type taskGetMultipleRowsOutput struct { + Rows []Row `instill:"rows"` +} + +// Row represents a row with row number and data +type Row struct { + RowNumber int `instill:"row-number"` + RowValue map[string]format.Value `instill:"row-value"` +} + +// taskInsertRowInput represents input for inserting a row +type taskInsertRowInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + RowValue map[string]format.Value `instill:"row-value"` +} + +// taskInsertRowOutput represents output after inserting a row +type taskInsertRowOutput struct { + Row Row `instill:"row"` +} + +// taskInsertMultipleRowsInput represents input for inserting multiple rows +type taskInsertMultipleRowsInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + RowValues []map[string]format.Value `instill:"row-values"` +} + +// taskInsertMultipleRowsOutput represents output after inserting multiple rows +type taskInsertMultipleRowsOutput struct { + Rows []Row `instill:"rows"` +} + +// taskUpdateRowInput represents input for updating a row +type taskUpdateRowInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + Row Row `instill:"row"` +} + +// taskUpdateRowOutput represents output after updating a row +type taskUpdateRowOutput struct { + Row Row `instill:"row"` +} + +// taskUpdateMultipleRowsInput represents input for updating multiple rows +type taskUpdateMultipleRowsInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + Rows []Row `instill:"rows"` +} + +// taskUpdateMultipleRowsOutput represents output after updating multiple rows +type taskUpdateMultipleRowsOutput struct { + Rows []Row `instill:"rows"` +} + +// taskDeleteRowInput represents input for deleting a row +type taskDeleteRowInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + RowNumber int `instill:"row-number"` +} + +// taskDeleteRowOutput represents output after deleting a row +type taskDeleteRowOutput struct { + Success bool `instill:"success"` +} + +// taskDeleteMultipleRowsInput represents input for deleting multiple rows +type taskDeleteMultipleRowsInput struct { + SharedLink string `instill:"shared-link"` + SheetName string `instill:"sheet-name"` + RowNumbers []int `instill:"row-numbers"` +} + +// taskDeleteMultipleRowsOutput represents output after deleting multiple rows +type taskDeleteMultipleRowsOutput struct { + Success bool `instill:"success"` +} diff --git a/pkg/component/data/googlesheets/v0/main.go b/pkg/component/data/googlesheets/v0/main.go new file mode 100644 index 000000000..2ecbb6af7 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/main.go @@ -0,0 +1,200 @@ +//go:generate compogen readme ./config ./README.mdx +package googlesheets + +import ( + "context" + "fmt" + "sync" + + _ "embed" + + "golang.org/x/oauth2" + "google.golang.org/api/drive/v2" + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +const ( + authURL = "https://accounts.google.com/o/oauth2/auth" + tokenURL = "https://oauth2.googleapis.com/token" +) + +var ( + //go:embed config/definition.json + definitionJSON []byte + //go:embed config/tasks.json + tasksJSON []byte + //go:embed config/setup.json + setupJSON []byte + + once sync.Once + comp *component +) + +const ( + taskCreateSpreadsheet = "TASK_CREATE_SPREADSHEET" + taskDeleteSpreadsheet = "TASK_DELETE_SPREADSHEET" + taskAddSheet = "TASK_ADD_SHEET" + taskDeleteSheet = "TASK_DELETE_SHEET" + taskCreateSpreadsheetColumn = "TASK_CREATE_SPREADSHEET_COLUMN" + taskDeleteSpreadsheetColumn = "TASK_DELETE_SPREADSHEET_COLUMN" + taskListRows = "TASK_LIST_ROWS" + taskLookupRows = "TASK_LOOKUP_ROWS" + taskGetRow = "TASK_GET_ROW" + taskGetMultipleRows = "TASK_GET_MULTIPLE_ROWS" + taskInsertRow = "TASK_INSERT_ROW" + taskInsertMultipleRows = "TASK_INSERT_MULTIPLE_ROWS" + taskUpdateRow = "TASK_UPDATE_ROW" + taskUpdateMultipleRows = "TASK_UPDATE_MULTIPLE_ROWS" + taskDeleteRow = "TASK_DELETE_ROW" + taskDeleteMultipleRows = "TASK_DELETE_MULTIPLE_ROWS" +) + +type component struct { + base.Component + base.OAuthConnector +} + +type execution struct { + base.ComponentExecution + execute func(context.Context, *base.Job) error + sheetService *sheets.Service + driveService *drive.Service +} + +func Init(bc base.Component) *component { + once.Do(func() { + comp = &component{Component: bc} + err := comp.LoadDefinition(definitionJSON, setupJSON, tasksJSON, nil, nil) + if err != nil { + panic(err) + } + }) + return comp +} + +func getsheetService(ctx context.Context, setup *structpb.Struct, c *component) (*sheets.Service, error) { + config := &oauth2.Config{ + ClientID: c.GetOAuthClientID(), + ClientSecret: c.GetOAuthClientSecret(), + Endpoint: oauth2.Endpoint{ + AuthURL: authURL, + TokenURL: tokenURL, + }, + } + + refreshToken := setup.GetFields()["refresh-token"].GetStringValue() + + tok := &oauth2.Token{ + RefreshToken: refreshToken, + } + + client := config.Client(ctx, tok) + + srv, err := sheets.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + return nil, err + } + + return srv, nil +} + +func getDriveService(ctx context.Context, setup *structpb.Struct, c *component) (*drive.Service, error) { + config := &oauth2.Config{ + ClientID: c.GetOAuthClientID(), + ClientSecret: c.GetOAuthClientSecret(), + Endpoint: oauth2.Endpoint{ + AuthURL: authURL, + TokenURL: tokenURL, + }, + } + + refreshToken := setup.GetFields()["refresh-token"].GetStringValue() + + tok := &oauth2.Token{ + RefreshToken: refreshToken, + } + + client := config.Client(ctx, tok) + + srv, err := drive.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + return nil, err + } + + return srv, nil + +} + +// CreateExecution initializes a component executor that can be used in a +// pipeline trigger. +func (c *component) CreateExecution(x base.ComponentExecution) (base.IExecution, error) { + ctx := context.Background() + + sheetSrv, err := getsheetService(ctx, x.Setup, c) + if err != nil { + return nil, fmt.Errorf("failed to get sheets service: %w", err) + } + driveSrv, err := getDriveService(ctx, x.Setup, c) + if err != nil { + return nil, fmt.Errorf("failed to get drive service: %w", err) + } + + e := &execution{ + ComponentExecution: x, + sheetService: sheetSrv, + driveService: driveSrv, + } + + switch x.Task { + case taskCreateSpreadsheet: + e.execute = e.createSpreadsheet + case taskDeleteSpreadsheet: + e.execute = e.deleteSpreadsheet + case taskAddSheet: + e.execute = e.addSheet + case taskDeleteSheet: + e.execute = e.deleteSheet + case taskCreateSpreadsheetColumn: + e.execute = e.createSpreadsheetColumn + case taskDeleteSpreadsheetColumn: + e.execute = e.deleteSpreadsheetColumn + case taskListRows: + e.execute = e.listRows + case taskLookupRows: + e.execute = e.lookupRows + case taskGetRow: + e.execute = e.getRow + case taskGetMultipleRows: + e.execute = e.getMultipleRows + case taskInsertRow: + e.execute = e.insertRow + case taskInsertMultipleRows: + e.execute = e.insertMultipleRows + case taskUpdateRow: + e.execute = e.updateRow + case taskUpdateMultipleRows: + e.execute = e.updateMultipleRows + case taskDeleteRow: + e.execute = e.deleteRow + case taskDeleteMultipleRows: + e.execute = e.deleteMultipleRows + default: + return nil, fmt.Errorf("not supported task: %s", x.Task) + } + + return e, nil +} + +// Execute executes the derived execution +func (e *execution) Execute(ctx context.Context, jobs []*base.Job) error { + return base.ConcurrentExecutor(ctx, jobs, e.execute) +} + +// SupportsOAuth checks whether the component is configured to support OAuth. +func (c *component) SupportsOAuth() bool { + return c.OAuthConnector.SupportsOAuth() +} diff --git a/pkg/component/data/googlesheets/v0/task_add_sheet.go b/pkg/component/data/googlesheets/v0/task_add_sheet.go new file mode 100644 index 000000000..d0d9e91bf --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_add_sheet.go @@ -0,0 +1,68 @@ +package googlesheets + +import ( + "context" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "google.golang.org/api/sheets/v4" +) + +func (e *execution) addSheet(ctx context.Context, job *base.Job) error { + input := &taskAddSheetInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + spreadsheetID, err := e.extractSpreadsheetID(input.SharedLink) + if err != nil { + return err + } + + // Create the add sheet request + addSheetRequest := &sheets.Request{ + AddSheet: &sheets.AddSheetRequest{ + Properties: &sheets.SheetProperties{ + Title: input.SheetName, + }, + }, + } + + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{addSheetRequest}, + } + + // Execute the batch update + _, err = e.sheetService.Spreadsheets.BatchUpdate(spreadsheetID, batchUpdateRequest).Context(ctx).Do() + if err != nil { + return err + } + + // If headers are provided, update the first row + if len(input.Headers) > 0 { + valueRange := &sheets.ValueRange{ + Values: [][]any{ + e.convertStringsToInterface(input.Headers), + }, + } + + // Update the header row + _, err = e.sheetService.Spreadsheets.Values.Update( + spreadsheetID, + input.SheetName+"!A1", + valueRange, + ).ValueInputOption("RAW").Context(ctx).Do() + if err != nil { + return err + } + } + + // TODO(huitang): reflect the real status + output := &taskAddSheetOutput{ + Success: true, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_add_sheet_test.go b/pkg/component/data/googlesheets/v0/task_add_sheet_test.go new file mode 100644 index 000000000..09b276df0 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_add_sheet_test.go @@ -0,0 +1,135 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestAddSheet(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id:batchUpdate" && r.Method == "POST" { + // Return mock batch update response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "spreadsheetId": "test-id", + "replies": [{"addSheet": {"properties": {"sheetId": 123, "title": "new-sheet"}}}] + }`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskAddSheetInput + expectedOutput taskAddSheetOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - add new sheet", + input: taskAddSheetInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "new-sheet", + }, + expectedOutput: taskAddSheetOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid shared link", + input: taskAddSheetInput{ + SharedLink: "invalid-link", + SheetName: "new-sheet", + }, + expectedOutput: taskAddSheetOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskAddSheet, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskAddSheetInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskAddSheetOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + switch output := output.(type) { + case *taskAddSheetOutput: + capturedOutput = *output + } + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_create_spreadsheet.go b/pkg/component/data/googlesheets/v0/task_create_spreadsheet.go new file mode 100644 index 000000000..9e9e19422 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_create_spreadsheet.go @@ -0,0 +1,81 @@ +package googlesheets + +import ( + "context" + + "google.golang.org/api/sheets/v4" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) createSpreadsheet(ctx context.Context, job *base.Job) error { + input := &taskCreateSpreadsheetInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + // Create a new spreadsheet + spreadsheet := &sheets.Spreadsheet{ + Properties: &sheets.SpreadsheetProperties{ + Title: input.Title, + }, + } + + // Create the spreadsheet + createdSpreadsheet, err := e.sheetService.Spreadsheets.Create(spreadsheet).Context(ctx).Do() + if err != nil { + return err + } + + // For each sheet in the input + for _, sheet := range input.Sheets { + if sheet.Name != "sheet1" { + // Create the add sheet request + addSheetRequest := &sheets.Request{ + AddSheet: &sheets.AddSheetRequest{ + Properties: &sheets.SheetProperties{ + Title: sheet.Name, + }, + }, + } + + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{addSheetRequest}, + } + + // Execute the batch update + _, err = e.sheetService.Spreadsheets.BatchUpdate(createdSpreadsheet.SpreadsheetId, batchUpdateRequest).Context(ctx).Do() + if err != nil { + return err + } + } + + // If headers are provided, update the first row + if len(sheet.Headers) > 0 { + valueRange := &sheets.ValueRange{ + Values: [][]any{ + e.convertStringsToInterface(sheet.Headers), + }, + } + + // Update the header row + _, err = e.sheetService.Spreadsheets.Values.Update( + createdSpreadsheet.SpreadsheetId, + sheet.Name+"!A1", + valueRange, + ).ValueInputOption("RAW").Context(ctx).Do() + if err != nil { + return err + } + } + } + + output := &taskCreateSpreadsheetOutput{ + SharedLink: createdSpreadsheet.SpreadsheetUrl, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_create_spreadsheet_column.go b/pkg/component/data/googlesheets/v0/task_create_spreadsheet_column.go new file mode 100644 index 000000000..1c727d02a --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_create_spreadsheet_column.go @@ -0,0 +1,69 @@ +package googlesheets + +import ( + "context" + + "google.golang.org/api/sheets/v4" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) createSpreadsheetColumn(ctx context.Context, job *base.Job) error { + input := &taskCreateSpreadsheetColumnInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + spreadsheetID, err := e.extractSpreadsheetID(input.SharedLink) + if err != nil { + return err + } + + // Get the current sheet data to find the last column + resp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + input.SheetName+"!1:1", + ).Context(ctx).Do() + if err != nil { + return err + } + + var columnIndex int + if len(resp.Values) > 0 { + columnIndex = len(resp.Values[0]) + 1 + } else { + columnIndex = 1 + } + + // Convert column index to A1 notation + columnLetter := "" + for columnIndex > 0 { + columnIndex-- + columnLetter = string(rune('A'+columnIndex%26)) + columnLetter + columnIndex = columnIndex / 26 + } + + // Update the header with the new column name + valueRange := &sheets.ValueRange{ + Values: [][]any{{input.ColumnName}}, + } + + _, err = e.sheetService.Spreadsheets.Values.Update( + spreadsheetID, + input.SheetName+"!"+columnLetter+"1", + valueRange, + ).ValueInputOption("RAW").Context(ctx).Do() + if err != nil { + return err + } + + // TODO(huitang): reflect the real status + output := &taskCreateSpreadsheetColumnOutput{ + Success: true, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_create_spreadsheet_column_test.go b/pkg/component/data/googlesheets/v0/task_create_spreadsheet_column_test.go new file mode 100644 index 000000000..0c91d06df --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_create_spreadsheet_column_test.go @@ -0,0 +1,167 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestCreateSpreadsheetColumn(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!1:1" && r.Method == "GET" { + // Return mock get values response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Header1", "Header2"]] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!C1" && r.Method == "PUT" { + // Return mock update values response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + if r.URL.Path == "/v4/spreadsheets/empty-sheet/values/sheet1!1:1" && r.Method == "GET" { + // Return mock empty sheet response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + if r.URL.Path == "/v4/spreadsheets/empty-sheet/values/sheet1!A1" && r.Method == "PUT" { + // Return mock update values response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskCreateSpreadsheetColumnInput + expectedOutput taskCreateSpreadsheetColumnOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - create column in existing sheet", + input: taskCreateSpreadsheetColumnInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + ColumnName: "Header3", + }, + expectedOutput: taskCreateSpreadsheetColumnOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "ok - create first column in empty sheet", + input: taskCreateSpreadsheetColumnInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/empty-sheet", + SheetName: "sheet1", + ColumnName: "Header1", + }, + expectedOutput: taskCreateSpreadsheetColumnOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskCreateSpreadsheetColumnInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + ColumnName: "Header1", + }, + expectedOutput: taskCreateSpreadsheetColumnOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskCreateSpreadsheetColumn, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskCreateSpreadsheetColumnInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskCreateSpreadsheetColumnOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + switch output := output.(type) { + case *taskCreateSpreadsheetColumnOutput: + capturedOutput = *output + } + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_create_spreadsheet_test.go b/pkg/component/data/googlesheets/v0/task_create_spreadsheet_test.go new file mode 100644 index 000000000..40ae8b667 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_create_spreadsheet_test.go @@ -0,0 +1,183 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestCreateSpreadsheet(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets" && r.Method == "POST" { + // Return mock spreadsheet response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "spreadsheetId": "test-id", + "spreadsheetUrl": "https://test-spreadsheet-url", + "sheets": [{"properties": {"title": "sheet1"}}] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id:batchUpdate" && r.Method == "POST" { + // Return mock batch update response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!A1" && r.Method == "PUT" { + // Return mock update values response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet2!A1" && r.Method == "PUT" { + // Return mock update values response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskCreateSpreadsheetInput + expectedOutput taskCreateSpreadsheetOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - create spreadsheet with single sheet", + input: taskCreateSpreadsheetInput{ + Title: "Test Spreadsheet", + Sheets: []sheet{ + { + Name: "sheet1", + Headers: []string{"Header1", "Header2"}, + }, + }, + }, + expectedOutput: taskCreateSpreadsheetOutput{ + SharedLink: "https://test-spreadsheet-url", + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "ok - create spreadsheet with multiple sheets", + input: taskCreateSpreadsheetInput{ + Title: "Test Spreadsheet Multiple", + Sheets: []sheet{ + { + Name: "sheet1", + Headers: []string{"Header1", "Header2"}, + }, + { + Name: "sheet2", + Headers: []string{"Header3", "Header4"}, + }, + }, + }, + expectedOutput: taskCreateSpreadsheetOutput{ + SharedLink: "https://test-spreadsheet-url", + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "ok - create spreadsheet without headers", + input: taskCreateSpreadsheetInput{ + Title: "Test Spreadsheet No Headers", + Sheets: []sheet{ + { + Name: "sheet1", + }, + }, + }, + expectedOutput: taskCreateSpreadsheetOutput{ + SharedLink: "https://test-spreadsheet-url", + }, + expectErr: false, + expectedErrMsg: "", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskCreateSpreadsheet, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskCreateSpreadsheetInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput *taskCreateSpreadsheetOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = output.(*taskCreateSpreadsheetOutput) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput.SharedLink, qt.Equals, tc.expectedOutput.SharedLink) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_multiple_rows.go b/pkg/component/data/googlesheets/v0/task_delete_multiple_rows.go new file mode 100644 index 000000000..d095b7e70 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_multiple_rows.go @@ -0,0 +1,72 @@ +package googlesheets + +import ( + "context" + + "google.golang.org/api/sheets/v4" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) deleteRowsHelper(ctx context.Context, sharedLink string, sheetName string, rowNumbers []int) error { + + spreadsheetID, err := e.extractSpreadsheetID(sharedLink) + if err != nil { + return err + } + + sheetID, err := e.convertSheetNameToSheetID(ctx, spreadsheetID, sheetName) + if err != nil { + return err + } + + // Create delete dimension request for each row index + var requests []*sheets.Request + for _, rowNumber := range rowNumbers { + requests = append(requests, &sheets.Request{ + DeleteDimension: &sheets.DeleteDimensionRequest{ + Range: &sheets.DimensionRange{ + SheetId: sheetID, + Dimension: "ROWS", + StartIndex: int64(rowNumber - 1), // Convert to 0-based index + EndIndex: int64(rowNumber), // Delete 1 row + }, + }, + }) + } + + // Execute batch update + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: requests, + } + + _, err = e.sheetService.Spreadsheets.BatchUpdate(spreadsheetID, batchUpdateRequest).Context(ctx).Do() + if err != nil { + return err + } + + return nil +} + +func (e *execution) deleteMultipleRows(ctx context.Context, job *base.Job) error { + input := &taskDeleteMultipleRowsInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + err := e.deleteRowsHelper(ctx, input.SharedLink, input.SheetName, input.RowNumbers) + if err != nil { + return err + } + + // TODO(huitang): reflect the real status + output := &taskDeleteMultipleRowsOutput{ + Success: true, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil + +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_multiple_rows_test.go b/pkg/component/data/googlesheets/v0/task_delete_multiple_rows_test.go new file mode 100644 index 000000000..9573ba1b3 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_multiple_rows_test.go @@ -0,0 +1,146 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestDeleteMultipleRows(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id" && r.Method == "GET" { + // Return mock spreadsheet response with sheet info + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "sheets": [ + { + "properties": { + "sheetId": 0, + "title": "sheet1" + } + } + ] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id:batchUpdate" && r.Method == "POST" { + // Return mock batch update response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskDeleteMultipleRowsInput + expectedOutput taskDeleteMultipleRowsOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - delete multiple rows", + input: taskDeleteMultipleRowsInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + RowNumbers: []int{2, 4}, + }, + expectedOutput: taskDeleteMultipleRowsOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskDeleteMultipleRowsInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + RowNumbers: []int{2, 4}, + }, + expectedOutput: taskDeleteMultipleRowsOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskDeleteMultipleRows, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskDeleteMultipleRowsInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskDeleteMultipleRowsOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskDeleteMultipleRowsOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_row.go b/pkg/component/data/googlesheets/v0/task_delete_row.go new file mode 100644 index 000000000..50bc13e5c --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_row.go @@ -0,0 +1,29 @@ +package googlesheets + +import ( + "context" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) deleteRow(ctx context.Context, job *base.Job) error { + input := &taskDeleteRowInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + err := e.deleteRowsHelper(ctx, input.SharedLink, input.SheetName, []int{input.RowNumber}) + if err != nil { + return err + } + + // TODO(huitang): reflect the real status + output := &taskDeleteRowOutput{ + Success: true, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_row_test.go b/pkg/component/data/googlesheets/v0/task_delete_row_test.go new file mode 100644 index 000000000..14892af81 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_row_test.go @@ -0,0 +1,146 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestDeleteRow(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id" && r.Method == "GET" { + // Return mock spreadsheet response with sheet info + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "sheets": [ + { + "properties": { + "sheetId": 0, + "title": "sheet1" + } + } + ] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id:batchUpdate" && r.Method == "POST" { + // Return mock batch update response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskDeleteRowInput + expectedOutput taskDeleteRowOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - delete single row", + input: taskDeleteRowInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + RowNumber: 1, + }, + expectedOutput: taskDeleteRowOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskDeleteRowInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + RowNumber: 1, + }, + expectedOutput: taskDeleteRowOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskDeleteRow, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskDeleteRowInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskDeleteRowOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskDeleteRowOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_sheet.go b/pkg/component/data/googlesheets/v0/task_delete_sheet.go new file mode 100644 index 000000000..3c97734c5 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_sheet.go @@ -0,0 +1,52 @@ +package googlesheets + +import ( + "context" + + "google.golang.org/api/sheets/v4" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) deleteSheet(ctx context.Context, job *base.Job) error { + input := &taskDeleteSheetInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + spreadsheetID, err := e.extractSpreadsheetID(input.SharedLink) + if err != nil { + return err + } + + sheetID, err := e.convertSheetNameToSheetID(ctx, spreadsheetID, input.SheetName) + if err != nil { + return err + } + + // Create delete sheet request + request := &sheets.Request{ + DeleteSheet: &sheets.DeleteSheetRequest{ + SheetId: sheetID, + }, + } + + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{request}, + } + + // Execute the batch update + _, err = e.sheetService.Spreadsheets.BatchUpdate(spreadsheetID, batchUpdateRequest).Context(ctx).Do() + if err != nil { + return err + } + + output := &taskDeleteSheetOutput{ + Success: true, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_sheet_test.go b/pkg/component/data/googlesheets/v0/task_delete_sheet_test.go new file mode 100644 index 000000000..cc613c2b8 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_sheet_test.go @@ -0,0 +1,144 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestDeleteSheet(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id" && r.Method == "GET" { + // Return mock spreadsheet response with sheet info + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "sheets": [ + { + "properties": { + "sheetId": 0, + "title": "sheet1" + } + } + ] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id:batchUpdate" && r.Method == "POST" { + // Return mock batch update response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskDeleteSheetInput + expectedOutput taskDeleteSheetOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - delete sheet", + input: taskDeleteSheetInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + }, + expectedOutput: taskDeleteSheetOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid shared link", + input: taskDeleteSheetInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + }, + expectedOutput: taskDeleteSheetOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskDeleteSheet, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskDeleteSheetInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskDeleteSheetOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskDeleteSheetOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_spreadsheet.go b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet.go new file mode 100644 index 000000000..aa8c92587 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet.go @@ -0,0 +1,34 @@ +package googlesheets + +import ( + "context" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) deleteSpreadsheet(ctx context.Context, job *base.Job) error { + input := &taskDeleteSpreadsheetInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + spreadsheetID, err := e.extractSpreadsheetID(input.SharedLink) + if err != nil { + return err + } + + // Delete the spreadsheet using Drive API + err = e.driveService.Files.Delete(spreadsheetID).Context(ctx).Do() + if err != nil { + return err + } + + output := &taskDeleteSpreadsheetOutput{ + Success: true, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_column.go b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_column.go new file mode 100644 index 000000000..dd544095b --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_column.go @@ -0,0 +1,82 @@ +package googlesheets + +import ( + "context" + "fmt" + + "google.golang.org/api/sheets/v4" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) deleteSpreadsheetColumn(ctx context.Context, job *base.Job) error { + input := &taskDeleteSpreadsheetColumnInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + spreadsheetID, err := e.extractSpreadsheetID(input.SharedLink) + if err != nil { + return err + } + + sheetID, err := e.convertSheetNameToSheetID(ctx, spreadsheetID, input.SheetName) + if err != nil { + return err + } + + // Get the current sheet data to find the column index + resp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + input.SheetName+"!1:1", + ).Context(ctx).Do() + if err != nil { + return err + } + + // Find the column index + var columnIndex int = -1 + if len(resp.Values) > 0 { + for i, header := range resp.Values[0] { + if header.(string) == input.ColumnName { + columnIndex = i + break + } + } + } + + if columnIndex == -1 { + return fmt.Errorf("column not found") + } + + // Create delete dimension request + request := &sheets.Request{ + DeleteDimension: &sheets.DeleteDimensionRequest{ + Range: &sheets.DimensionRange{ + SheetId: sheetID, + Dimension: "COLUMNS", + StartIndex: int64(columnIndex), + EndIndex: int64(columnIndex + 1), // Delete 1 column + }, + }, + } + + // Execute batch update + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{request}, + } + + _, err = e.sheetService.Spreadsheets.BatchUpdate(spreadsheetID, batchUpdateRequest).Context(ctx).Do() + if err != nil { + return err + } + + output := &taskDeleteSpreadsheetColumnOutput{ + Success: true, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_column_test.go b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_column_test.go new file mode 100644 index 000000000..561470338 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_column_test.go @@ -0,0 +1,174 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestDeleteSpreadsheetColumn(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id" && r.Method == "GET" { + // Return mock get spreadsheet response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "sheets": [ + { + "properties": { + "sheetId": 0, + "title": "sheet1" + } + } + ] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id:batchUpdate" && r.Method == "POST" { + // Return mock batch update response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!1:1" && r.Method == "GET" { + // Return mock get values response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Header1", "Header2", "Header3"]] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!C1" && r.Method == "PUT" { + // Return mock update values response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskDeleteSpreadsheetColumnInput + expectedOutput taskDeleteSpreadsheetColumnOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - delete column from sheet", + input: taskDeleteSpreadsheetColumnInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + ColumnName: "Header3", + }, + expectedOutput: taskDeleteSpreadsheetColumnOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskDeleteSpreadsheetColumnInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + ColumnName: "Header1", + }, + expectedOutput: taskDeleteSpreadsheetColumnOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + { + name: "error - column not found", + input: taskDeleteSpreadsheetColumnInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + ColumnName: "NonExistentHeader", + }, + expectedOutput: taskDeleteSpreadsheetColumnOutput{}, + expectErr: true, + expectedErrMsg: "column not found", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskDeleteSpreadsheetColumn, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskDeleteSpreadsheetColumnInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskDeleteSpreadsheetColumnOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + switch output := output.(type) { + case *taskDeleteSpreadsheetColumnOutput: + capturedOutput = *output + } + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_test.go b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_test.go new file mode 100644 index 000000000..5dfc4f0fb --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_delete_spreadsheet_test.go @@ -0,0 +1,126 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/drive/v2" + "google.golang.org/api/option" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" +) + +func TestDeleteSpreadsheet(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/files/test-id" && r.Method == "DELETE" { + // Return mock delete response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskDeleteSpreadsheetInput + expectedOutput taskDeleteSpreadsheetOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - delete spreadsheet", + input: taskDeleteSpreadsheetInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + }, + expectedOutput: taskDeleteSpreadsheetOutput{ + Success: true, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskDeleteSpreadsheetInput{ + SharedLink: "invalid-link", + }, + expectedOutput: taskDeleteSpreadsheetOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + driveService, err := drive.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskDeleteSpreadsheet, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).driveService = driveService + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskDeleteSpreadsheetInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskDeleteSpreadsheetOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskDeleteSpreadsheetOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_get_multiple_rows.go b/pkg/component/data/googlesheets/v0/task_get_multiple_rows.go new file mode 100644 index 000000000..54aecec9e --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_get_multiple_rows.go @@ -0,0 +1,93 @@ +package googlesheets + +import ( + "context" + "fmt" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func (e *execution) getRowsHelper(ctx context.Context, sharedLink string, sheetName string, rowNumbers []int) ([]Row, error) { + spreadsheetID, err := e.extractSpreadsheetID(sharedLink) + if err != nil { + return nil, err + } + + // Get headers from first row + headerResp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + sheetName+"!1:1", + ).Context(ctx).Do() + if err != nil { + return nil, err + } + + if len(headerResp.Values) == 0 { + return nil, nil // Empty sheet + } + + headers := headerResp.Values[0] + + result := make([]Row, len(rowNumbers)) + for i, rowNum := range rowNumbers { + if rowNum <= 0 { + continue + } + + // Convert row number to A1 notation (e.g. "A5:Z5" for row 5) + rowRange := fmt.Sprintf("%s!%d:%d", sheetName, rowNum, rowNum) + + rowResp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + rowRange, + ).Context(ctx).Do() + if err != nil { + continue // Skip invalid rows + } + + if len(rowResp.Values) == 0 { + continue // Skip empty rows + } + + row := rowResp.Values[0] + rowMap := make(map[string]format.Value) + + for j, header := range headers { + headerStr := header.(string) + if j < len(row) && row[j] != nil { + switch r := row[j].(type) { + case string: + rowMap[headerStr] = data.NewString(r) + case float64: + rowMap[headerStr] = data.NewNumberFromFloat(r) + case bool: + rowMap[headerStr] = data.NewBoolean(r) + } + } + } + result[i].RowValue = rowMap + result[i].RowNumber = rowNum + } + + return result, nil +} + +func (e *execution) getMultipleRows(ctx context.Context, job *base.Job) error { + input := &taskGetMultipleRowsInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + rows, err := e.getRowsHelper(ctx, input.SharedLink, input.SheetName, input.RowNumbers) + if err != nil { + return err + } + + output := &taskGetMultipleRowsOutput{ + Rows: rows, + } + + return job.Output.WriteData(ctx, output) +} diff --git a/pkg/component/data/googlesheets/v0/task_get_multiple_rows_test.go b/pkg/component/data/googlesheets/v0/task_get_multiple_rows_test.go new file mode 100644 index 000000000..149d5b37a --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_get_multiple_rows_test.go @@ -0,0 +1,168 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func TestGetMultipleRows(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!1:1" && r.Method == "GET" { + // Return mock headers response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Header1", "Header2", "Header3"]] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!2:2" && r.Method == "GET" { + // Return mock row 2 response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Value1", "Value2", "Value3"]] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!3:3" && r.Method == "GET" { + // Return mock row 3 response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Value4", "Value5", "Value6"]] + }`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskGetMultipleRowsInput + expectedOutput taskGetMultipleRowsOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - get multiple rows", + input: taskGetMultipleRowsInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + RowNumbers: []int{2, 3}, + }, + expectedOutput: taskGetMultipleRowsOutput{ + Rows: []Row{ + { + RowNumber: 2, + RowValue: map[string]format.Value{ + "Header1": data.NewString("Value1"), + "Header2": data.NewString("Value2"), + "Header3": data.NewString("Value3"), + }, + }, + { + RowNumber: 3, + RowValue: map[string]format.Value{ + "Header1": data.NewString("Value4"), + "Header2": data.NewString("Value5"), + "Header3": data.NewString("Value6"), + }, + }, + }, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskGetMultipleRowsInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + RowNumbers: []int{1}, + }, + expectedOutput: taskGetMultipleRowsOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskGetMultipleRows, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskGetMultipleRowsInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskGetMultipleRowsOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskGetMultipleRowsOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_get_row.go b/pkg/component/data/googlesheets/v0/task_get_row.go new file mode 100644 index 000000000..8cd3f7cdf --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_get_row.go @@ -0,0 +1,25 @@ +package googlesheets + +import ( + "context" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) getRow(ctx context.Context, job *base.Job) error { + input := &taskGetRowInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + rows, err := e.getRowsHelper(ctx, input.SharedLink, input.SheetName, []int{input.RowNumber}) + if err != nil { + return err + } + + output := &taskGetRowOutput{ + Row: rows[0], + } + + return job.Output.WriteData(ctx, output) +} diff --git a/pkg/component/data/googlesheets/v0/task_get_row_test.go b/pkg/component/data/googlesheets/v0/task_get_row_test.go new file mode 100644 index 000000000..5c7285c88 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_get_row_test.go @@ -0,0 +1,150 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func TestGetRow(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!1:1" && r.Method == "GET" { + // Return mock headers response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Header1", "Header2", "Header3"]] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!2:2" && r.Method == "GET" { + // Return mock row response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Value1", "Value2", "Value3"]] + }`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskGetRowInput + expectedOutput taskGetRowOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - get row", + input: taskGetRowInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + RowNumber: 2, + }, + expectedOutput: taskGetRowOutput{ + Row: Row{ + RowNumber: 2, + RowValue: map[string]format.Value{ + "Header1": data.NewString("Value1"), + "Header2": data.NewString("Value2"), + "Header3": data.NewString("Value3"), + }, + }, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskGetRowInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + RowNumber: 1, + }, + expectedOutput: taskGetRowOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskGetRow, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskGetRowInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskGetRowOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskGetRowOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_insert_multiple_rows.go b/pkg/component/data/googlesheets/v0/task_insert_multiple_rows.go new file mode 100644 index 000000000..2fc3c7568 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_insert_multiple_rows.go @@ -0,0 +1,159 @@ +package googlesheets + +import ( + "context" + "fmt" + + "google.golang.org/api/sheets/v4" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func (e *execution) insertRowsHelper(ctx context.Context, sharedLink string, sheetName string, rows []map[string]format.Value) ([]Row, error) { + + spreadsheetID, err := e.extractSpreadsheetID(sharedLink) + if err != nil { + return nil, err + } + + sheetID, err := e.convertSheetNameToSheetID(ctx, spreadsheetID, sheetName) + if err != nil { + return nil, err + } + + // Get the last row index by querying the sheet data + resp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + sheetName, + ).Context(ctx).Do() + if err != nil { + return nil, err + } + + // Get the header row + if len(resp.Values) == 0 { + return nil, nil // Empty sheet, no headers + } + headers := resp.Values[0] // First row contains headers + + // Create values array for each row + var values [][]any + for _, row := range rows { + rowValues := make([]any, len(headers)) + for colIdx, header := range headers { + headerStr, ok := header.(string) + if !ok { + continue + } + if val, exists := row[headerStr]; exists { + rowValues[colIdx] = val.String() + } + } + values = append(values, rowValues) + } + + // Create insert dimension request + request := &sheets.Request{ + AppendCells: &sheets.AppendCellsRequest{ + SheetId: sheetID, + Rows: []*sheets.RowData{}, + Fields: "*", + }, + } + + // Add each row to the request + for _, rowValues := range values { + cells := make([]*sheets.CellData, len(headers)) + for colIdx, value := range rowValues { + if value == nil { + continue + } + valueStr := value.(string) + cells[colIdx] = &sheets.CellData{ + UserEnteredValue: &sheets.ExtendedValue{ + StringValue: &valueStr, + }, + } + } + request.AppendCells.Rows = append(request.AppendCells.Rows, &sheets.RowData{ + Values: cells, + }) + } + + // Execute batch update + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{request}, + } + + _, err = e.sheetService.Spreadsheets.BatchUpdate(spreadsheetID, batchUpdateRequest).Context(ctx).Do() + if err != nil { + return nil, err + } + + // Get the last row number before insertion + lastRowResp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + fmt.Sprintf("%s!A1:A", sheetName), + ).Context(ctx).Do() + if err != nil { + return nil, err + } + + startRow := len(lastRowResp.Values) - len(values) + 1 + rowNumbers := make([]int, len(values)) + for i := range values { + rowNumbers[i] = startRow + i + } + + // Convert the inserted values back to map format + insertedRows := make([]Row, len(values)) + for i, rowValues := range values { + rowMap := make(map[string]format.Value) + for j, val := range rowValues { + if j >= len(headers) { + continue + } + headerStr, ok := headers[j].(string) + if !ok { + continue + } + switch val := val.(type) { + case string: + rowMap[headerStr] = data.NewString(val) + case float64: + rowMap[headerStr] = data.NewNumberFromFloat(val) + case bool: + rowMap[headerStr] = data.NewBoolean(val) + } + } + insertedRows[i] = Row{ + RowValue: rowMap, + RowNumber: rowNumbers[i], + } + } + + return insertedRows, nil +} + +func (e *execution) insertMultipleRows(ctx context.Context, job *base.Job) error { + input := &taskInsertMultipleRowsInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + insertedRows, err := e.insertRowsHelper(ctx, input.SharedLink, input.SheetName, input.RowValues) + if err != nil { + return err + } + + output := &taskInsertMultipleRowsOutput{ + Rows: insertedRows, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_insert_multiple_rows_test.go b/pkg/component/data/googlesheets/v0/task_insert_multiple_rows_test.go new file mode 100644 index 000000000..90449afda --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_insert_multiple_rows_test.go @@ -0,0 +1,3 @@ +package googlesheets + +// TODO: add tests diff --git a/pkg/component/data/googlesheets/v0/task_insert_row.go b/pkg/component/data/googlesheets/v0/task_insert_row.go new file mode 100644 index 000000000..01249c0ba --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_insert_row.go @@ -0,0 +1,29 @@ +package googlesheets + +import ( + "context" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func (e *execution) insertRow(ctx context.Context, job *base.Job) error { + input := &taskInsertRowInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + insertedRows, err := e.insertRowsHelper(ctx, input.SharedLink, input.SheetName, []map[string]format.Value{input.RowValue}) + if err != nil { + return err + } + + output := &taskInsertRowOutput{ + Row: insertedRows[0], + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_insert_row_test.go b/pkg/component/data/googlesheets/v0/task_insert_row_test.go new file mode 100644 index 000000000..90449afda --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_insert_row_test.go @@ -0,0 +1,3 @@ +package googlesheets + +// TODO: add tests diff --git a/pkg/component/data/googlesheets/v0/task_list_rows.go b/pkg/component/data/googlesheets/v0/task_list_rows.go new file mode 100644 index 000000000..916a546ae --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_list_rows.go @@ -0,0 +1,84 @@ +package googlesheets + +import ( + "context" + "fmt" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func (e *execution) listRowsHelper(ctx context.Context, sharedLink string, sheetName string, startRow int, endRow int) ([]Row, error) { + spreadsheetID, err := e.extractSpreadsheetID(sharedLink) + if err != nil { + return nil, err + } + + // Get headers from first row + headerResp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + sheetName+"!1:1", + ).Context(ctx).Do() + if err != nil { + return nil, err + } + + if len(headerResp.Values) == 0 { + return nil, nil // Empty sheet + } + + headers := headerResp.Values[0] + + // Get data rows + dataRange := fmt.Sprintf("%s!%d:%d", sheetName, startRow, endRow) + dataResp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + dataRange, + ).Context(ctx).Do() + if err != nil { + return nil, err + } + + rows := make([]Row, 0) + for i, row := range dataResp.Values { + rowMap := make(map[string]format.Value) + + for j, header := range headers { + headerStr := header.(string) + if j < len(row) && row[j] != nil { + switch r := row[j].(type) { + case string: + rowMap[headerStr] = data.NewString(r) + case float64: + rowMap[headerStr] = data.NewNumberFromFloat(r) + case bool: + rowMap[headerStr] = data.NewBoolean(r) + } + } + } + rows = append(rows, Row{ + RowValue: rowMap, + RowNumber: i + startRow, + }) + } + return rows, nil +} + +func (e *execution) listRows(ctx context.Context, job *base.Job) error { + input := &taskListRowsInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + rows, err := e.listRowsHelper(ctx, input.SharedLink, input.SheetName, input.StartRow, input.EndRow) + if err != nil { + return err + } + + output := &taskListRowsOutput{ + Rows: rows, + } + + return job.Output.WriteData(ctx, output) +} diff --git a/pkg/component/data/googlesheets/v0/task_list_rows_test.go b/pkg/component/data/googlesheets/v0/task_list_rows_test.go new file mode 100644 index 000000000..bf3458b91 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_list_rows_test.go @@ -0,0 +1,175 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func TestListRows(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!1:1" && r.Method == "GET" { + // Return mock headers response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Header1", "Header2", "Header3"]] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!2:3" && r.Method == "GET" { + // Return mock headers response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [ + ["Value1", "Value2", "Value3"], + ["Value4", "Value5", "Value6"] + ] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1" && r.Method == "GET" { + // Return mock sheet data response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [ + ["Header1", "Header2", "Header3"], + ["Value1", "Value2", "Value3"], + ["Value4", "Value5", "Value6"] + ] + }`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskListRowsInput + expectedOutput taskListRowsOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - list all rows", + input: taskListRowsInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + StartRow: 2, + EndRow: 3, + }, + expectedOutput: taskListRowsOutput{ + Rows: []Row{ + { + RowNumber: 2, + RowValue: map[string]format.Value{ + "Header1": data.NewString("Value1"), + "Header2": data.NewString("Value2"), + "Header3": data.NewString("Value3"), + }, + }, + { + RowNumber: 3, + RowValue: map[string]format.Value{ + "Header1": data.NewString("Value4"), + "Header2": data.NewString("Value5"), + "Header3": data.NewString("Value6"), + }, + }, + }, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskListRowsInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + }, + expectedOutput: taskListRowsOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskListRows, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskListRowsInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskListRowsOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskListRowsOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_lookup_rows.go b/pkg/component/data/googlesheets/v0/task_lookup_rows.go new file mode 100644 index 000000000..f34923657 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_lookup_rows.go @@ -0,0 +1,93 @@ +package googlesheets + +import ( + "context" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func (e *execution) lookupRowsHelper(ctx context.Context, sharedLink string, sheetName string, columnName string, value string) ([]Row, error) { + spreadsheetID, err := e.extractSpreadsheetID(sharedLink) + if err != nil { + return nil, err + } + + // Get all values from the sheet + resp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + sheetName, + ).Context(ctx).Do() + if err != nil { + return nil, err + } + + if len(resp.Values) == 0 { + return nil, nil // Empty sheet + } + + // Get headers from first row + headers := resp.Values[0] + + // Find the target column index + var columnIndex int = -1 + for i, header := range headers { + if header.(string) == columnName { + columnIndex = i + break + } + } + + if columnIndex == -1 { + return nil, nil // Column not found + } + + // Look for matching rows + var result []Row + for i := 1; i < len(resp.Values); i++ { + row := resp.Values[i] + // Check if the column value matches exactly + if len(row) > columnIndex && row[columnIndex] != nil && row[columnIndex].(string) == value { + // Create map for matching row + rowMap := make(map[string]format.Value) + for j, header := range headers { + headerStr := header.(string) + if j < len(row) && row[j] != nil { + switch r := row[j].(type) { + case string: + rowMap[headerStr] = data.NewString(r) + case float64: + rowMap[headerStr] = data.NewNumberFromFloat(r) + case bool: + rowMap[headerStr] = data.NewBoolean(r) + } + } + } + result = append(result, Row{ + RowValue: rowMap, + RowNumber: i + 1, + }) + } + } + + return result, nil +} + +func (e *execution) lookupRows(ctx context.Context, job *base.Job) error { + input := &taskLookupRowsInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + rows, err := e.lookupRowsHelper(ctx, input.SharedLink, input.SheetName, input.ColumnName, input.Value) + if err != nil { + return err + } + + output := &taskLookupRowsOutput{ + Rows: rows, + } + + return job.Output.WriteData(ctx, output) +} diff --git a/pkg/component/data/googlesheets/v0/task_lookup_rows_test.go b/pkg/component/data/googlesheets/v0/task_lookup_rows_test.go new file mode 100644 index 000000000..11eac8b19 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_lookup_rows_test.go @@ -0,0 +1,159 @@ +package googlesheets + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + + qt "github.com/frankban/quicktest" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/component/internal/mock" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func TestLookupRows(t *testing.T) { + c := qt.New(t) + + // Create test server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1!1:1" && r.Method == "GET" { + // Return mock headers response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [["Header1", "Header2", "Header3"]] + }`)) + return + } + if r.URL.Path == "/v4/spreadsheets/test-id/values/sheet1" && r.Method == "GET" { + // Return mock all rows response + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "values": [ + ["Header1", "Header2", "Header3"], + ["Value1", "Value2", "Value3"], + ["Value4", "Value5", "Value6"], + ["Value7", "Value8", "Value9"] + ] + }`)) + return + } + http.Error(w, fmt.Sprintf("not found: %s %s", r.Method, r.URL.Path), http.StatusNotFound) + })) + defer ts.Close() + + testCases := []struct { + name string + input taskLookupRowsInput + expectedOutput taskLookupRowsOutput + expectErr bool + expectedErrMsg string + }{ + { + name: "ok - lookup rows", + input: taskLookupRowsInput{ + SharedLink: "https://docs.google.com/spreadsheets/d/test-id", + SheetName: "sheet1", + ColumnName: "Header2", + Value: "Value2", + }, + expectedOutput: taskLookupRowsOutput{ + Rows: []Row{ + { + RowNumber: 2, + RowValue: map[string]format.Value{ + "Header1": data.NewString("Value1"), + "Header2": data.NewString("Value2"), + "Header3": data.NewString("Value3"), + }, + }, + }, + }, + expectErr: false, + expectedErrMsg: "", + }, + { + name: "error - invalid spreadsheet ID", + input: taskLookupRowsInput{ + SharedLink: "invalid-link", + SheetName: "sheet1", + ColumnName: "Header1", + Value: "Value1", + }, + expectedOutput: taskLookupRowsOutput{}, + expectErr: true, + expectedErrMsg: "invalid shared link", + }, + } + + for _, tc := range testCases { + c.Run(tc.name, func(c *qt.C) { + // Create sheets service with test server + sheetsService, err := sheets.NewService(context.Background(), option.WithoutAuthentication(), option.WithEndpoint(ts.URL)) + c.Assert(err, qt.IsNil) + + component := Init(base.Component{}) + c.Assert(component, qt.IsNotNil) + + exe, err := component.CreateExecution(base.ComponentExecution{ + Component: component, + Task: taskLookupRows, + }) + c.Assert(err, qt.IsNil) + c.Assert(exe, qt.IsNotNil) + + // Set sheets service + exe.(*execution).sheetService = sheetsService + + // Generate mock job + ir, ow, eh, job := mock.GenerateMockJob(c) + + // Set up input mock + ir.ReadDataMock.Set(func(ctx context.Context, input any) error { + switch input := input.(type) { + case *taskLookupRowsInput: + *input = tc.input + } + return nil + }) + + // Set up output capture + var capturedOutput taskLookupRowsOutput + ow.WriteDataMock.Set(func(ctx context.Context, output any) error { + capturedOutput = *(output.(*taskLookupRowsOutput)) + return nil + }) + + // Set up error handling + var executionErr error + eh.ErrorMock.Set(func(ctx context.Context, err error) { + executionErr = err + }) + + if tc.expectErr { + ow.WriteDataMock.Optional() + } else { + eh.ErrorMock.Optional() + } + + // Execute the test + err = exe.Execute(context.Background(), []*base.Job{job}) + + if tc.expectErr { + c.Assert(executionErr, qt.Not(qt.IsNil)) + if tc.expectedErrMsg != "" { + c.Assert(executionErr.Error(), qt.Equals, tc.expectedErrMsg) + } + } else { + c.Assert(err, qt.IsNil) + c.Assert(capturedOutput, qt.DeepEquals, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/component/data/googlesheets/v0/task_update_multiple_rows.go b/pkg/component/data/googlesheets/v0/task_update_multiple_rows.go new file mode 100644 index 000000000..5ff27ec16 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_update_multiple_rows.go @@ -0,0 +1,155 @@ +package googlesheets + +import ( + "context" + "fmt" + + "google.golang.org/api/sheets/v4" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" + "github.com/instill-ai/pipeline-backend/pkg/data" + "github.com/instill-ai/pipeline-backend/pkg/data/format" +) + +func (e *execution) updateRowsHelper(ctx context.Context, sharedLink string, sheetName string, rows []Row) ([]Row, error) { + spreadsheetID, err := e.extractSpreadsheetID(sharedLink) + if err != nil { + return nil, err + } + + sheetID, err := e.convertSheetNameToSheetID(ctx, spreadsheetID, sheetName) + if err != nil { + return nil, err + } + + // Get the sheet data + resp, err := e.sheetService.Spreadsheets.Values.Get( + spreadsheetID, + sheetName, + ).Context(ctx).Do() + if err != nil { + return nil, err + } + + // Get the header row + if len(resp.Values) == 0 { + return nil, nil // Empty sheet, no headers + } + headers := resp.Values[0] // First row contains headers + + var requests []*sheets.Request + + // Create update requests for each row + for _, row := range rows { + + for colIdx, header := range headers { + headerStr, ok := header.(string) + if !ok { + continue + } + + if val, exists := row.RowValue[headerStr]; exists { + // Only add cell data if key exists in input row + valueStr := val.String() + cell := &sheets.CellData{ + UserEnteredValue: &sheets.ExtendedValue{ + StringValue: &valueStr, + }, + } + + request := &sheets.Request{ + UpdateCells: &sheets.UpdateCellsRequest{ + Range: &sheets.GridRange{ + SheetId: sheetID, + StartRowIndex: int64(row.RowNumber - 1), // Convert to 0-based index + EndRowIndex: int64(row.RowNumber), + StartColumnIndex: int64(colIdx), + EndColumnIndex: int64(colIdx + 1), + }, + Rows: []*sheets.RowData{ + { + Values: []*sheets.CellData{cell}, + }, + }, + Fields: "userEnteredValue", + }, + } + requests = append(requests, request) + } + } + + } + + // Execute batch update + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: requests, + } + + _, err = e.sheetService.Spreadsheets.BatchUpdate(spreadsheetID, batchUpdateRequest).Context(ctx).Do() + if err != nil { + return nil, err + } + + // Fetch the updated rows from Google Sheets + updatedRows := make([]Row, len(rows)) + for i, row := range rows { + // Get the specific row + rowRange := fmt.Sprintf("%s!%d:%d", sheetName, row.RowNumber, row.RowNumber) + rowResp, err := e.sheetService.Spreadsheets.Values.Get(spreadsheetID, rowRange).Context(ctx).Do() + if err != nil { + return nil, err + } + + if len(rowResp.Values) == 0 { + continue + } + + // Convert row data to map + rowMap := make(map[string]format.Value) + rowValues := rowResp.Values[0] + for j, header := range headers { + headerStr, ok := header.(string) + if !ok || j >= len(rowValues) { + continue + } + + if rowValues[j] != nil { + switch v := rowValues[j].(type) { + case string: + rowMap[headerStr] = data.NewString(v) + case float64: + rowMap[headerStr] = data.NewNumberFromFloat(v) + case bool: + rowMap[headerStr] = data.NewBoolean(v) + } + } + } + updatedRows[i] = Row{ + RowValue: rowMap, + RowNumber: row.RowNumber, + } + } + + return updatedRows, nil +} + +func (e *execution) updateMultipleRows(ctx context.Context, job *base.Job) error { + input := &taskUpdateMultipleRowsInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + updatedRows, err := e.updateRowsHelper(ctx, input.SharedLink, input.SheetName, input.Rows) + if err != nil { + return err + } + + output := &taskUpdateMultipleRowsOutput{ + Rows: updatedRows, + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_update_multiple_rows_test.go b/pkg/component/data/googlesheets/v0/task_update_multiple_rows_test.go new file mode 100644 index 000000000..90449afda --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_update_multiple_rows_test.go @@ -0,0 +1,3 @@ +package googlesheets + +// TODO: add tests diff --git a/pkg/component/data/googlesheets/v0/task_update_row.go b/pkg/component/data/googlesheets/v0/task_update_row.go new file mode 100644 index 000000000..4f6922d06 --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_update_row.go @@ -0,0 +1,28 @@ +package googlesheets + +import ( + "context" + + "github.com/instill-ai/pipeline-backend/pkg/component/base" +) + +func (e *execution) updateRow(ctx context.Context, job *base.Job) error { + input := &taskUpdateRowInput{} + if err := job.Input.ReadData(ctx, input); err != nil { + return err + } + + updatedRows, err := e.updateRowsHelper(ctx, input.SharedLink, input.SheetName, []Row{input.Row}) + if err != nil { + return err + } + + output := &taskUpdateRowOutput{ + Row: updatedRows[0], + } + if err := job.Output.WriteData(ctx, output); err != nil { + return err + } + + return nil +} diff --git a/pkg/component/data/googlesheets/v0/task_update_row_test.go b/pkg/component/data/googlesheets/v0/task_update_row_test.go new file mode 100644 index 000000000..90449afda --- /dev/null +++ b/pkg/component/data/googlesheets/v0/task_update_row_test.go @@ -0,0 +1,3 @@ +package googlesheets + +// TODO: add tests diff --git a/pkg/component/data/googlesheets/v0/utils.go b/pkg/component/data/googlesheets/v0/utils.go new file mode 100644 index 000000000..54b49295d --- /dev/null +++ b/pkg/component/data/googlesheets/v0/utils.go @@ -0,0 +1,41 @@ +package googlesheets + +import ( + "context" + "fmt" + "regexp" + "strings" +) + +func (e *execution) extractSpreadsheetID(sharedLink string) (string, error) { + re := regexp.MustCompile(`https://docs.google.com/spreadsheets/d/([a-zA-Z0-9-_]+)`) + matches := re.FindStringSubmatch(sharedLink) + if len(matches) < 2 { + return "", fmt.Errorf("invalid shared link") + } + + return matches[1], nil +} + +func (e *execution) convertStringsToInterface(strings []string) []any { + interfaces := make([]any, len(strings)) + for i, s := range strings { + interfaces[i] = s + } + return interfaces +} + +func (e *execution) convertSheetNameToSheetID(ctx context.Context, spreadsheetID string, sheetName string) (int64, error) { + spreadsheet, err := e.sheetService.Spreadsheets.Get(spreadsheetID).Context(ctx).Do() + if err != nil { + return 0, err + } + + for _, sheet := range spreadsheet.Sheets { + if strings.EqualFold(sheet.Properties.Title, sheetName) { + return sheet.Properties.SheetId, nil + } + } + + return 0, fmt.Errorf("sheet not found") +} diff --git a/pkg/component/store/store.go b/pkg/component/store/store.go index 81771f20b..4e360472e 100644 --- a/pkg/component/store/store.go +++ b/pkg/component/store/store.go @@ -40,6 +40,7 @@ import ( "github.com/instill-ai/pipeline-backend/pkg/component/data/elasticsearch/v0" "github.com/instill-ai/pipeline-backend/pkg/component/data/googlecloudstorage/v0" "github.com/instill-ai/pipeline-backend/pkg/component/data/googledrive/v0" + "github.com/instill-ai/pipeline-backend/pkg/component/data/googlesheets/v0" "github.com/instill-ai/pipeline-backend/pkg/component/data/instillartifact/v0" "github.com/instill-ai/pipeline-backend/pkg/component/data/milvus/v0" "github.com/instill-ai/pipeline-backend/pkg/component/data/mongodb/v0" @@ -184,6 +185,7 @@ func Init( compStore.Import(bigquery.Init(baseComp)) compStore.Import(googlecloudstorage.Init(baseComp)) compStore.Import(googlesearch.Init(baseComp)) + compStore.Import(pinecone.Init(baseComp)) compStore.Import(redis.Init(baseComp)) compStore.Import(elasticsearch.Init(baseComp)) @@ -216,6 +218,12 @@ func Init( conn.WithOAuthConfig(secrets["googledrive"]) compStore.Import(conn) } + { + // Google Sheets + conn := googlesheets.Init(baseComp) + conn.WithOAuthConfig(secrets["googlesheets"]) + compStore.Import(conn) + } compStore.Import(email.Init(baseComp)) compStore.Import(jira.Init(baseComp)) compStore.Import(ollama.Init(baseComp))