hubspot Destination

.github/workflows/build.yml

@@ -12,9 +12,9 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: "1.20"
 
       - name: Test
         run: make test GOTEST_FLAGS="-v -count=1 -race"

.github/workflows/lint.yml

@@ -9,11 +9,12 @@ jobs:
   golangci-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.18'
+          go-version: "1.20"
       - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.3.1
+        uses: golangci/golangci-lint-action@v3.6.0
         with:
-          version: v1.49.0
+          version: v1.53.3
+          args: --timeout=2m

.github/workflows/release.yml

@@ -19,9 +19,9 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: "1.20"
 
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4

.golangci.yml

@@ -2,7 +2,7 @@ linters-settings:
   gofmt:
     simplify: false
   govet:
-    check-shadowing: false
+    check-shadowing: true
   nolintlint:
     allow-unused: false # report any unused nolint directives
     require-explanation: true # require an explanation for nolint directives
@@ -25,7 +25,7 @@ linters:
   disable-all: true
   enable:
     - bodyclose
-    - depguard
+    # - depguard
     - dogsled
     - durationcheck
     - errcheck
@@ -61,10 +61,10 @@ linters:
     - nakedret
     - nilerr
     - nilnil
-    - nlreturn
+    # - nlreturn
     - noctx
     - nolintlint
-    - paralleltest
+    # - paralleltest
     - predeclared
     - rowserrcheck
     - staticcheck
@@ -101,4 +101,29 @@ linters:
     # - prealloc
     # - testpackage
     # - wsl
-    # - cyclop # not interested in package complexities at the moment
+    # - cyclop # not interested in package complexities at the moment
+
+# List of regexps of issue texts to exclude.
+issues:
+  exclude-rules:
+    - path: _test
+      linters:
+        - varnamelen
+        - testpackage
+        - nonamedreturns
+        - funlen # it can be annoying for table-driven tests
+        - dupl
+        - goerr113
+        - maintidx
+        - wrapcheck
+        - forcetypeassert
+    - path: _integration_test
+      linters:
+        - varnamelen
+        - testpackage
+        - nonamedreturns
+        - funlen # it can be annoying for table-driven tests
+        - dupl
+        - goerr113
+        - maintidx
+        - paralleltest # we don't want to run the integration tests in parallel because we want deterministic results

.goreleaser.yml

@@ -11,12 +11,13 @@ builds:
 checksum:
   name_template: checksums.txt
 archives:
-  - replacements:
-      darwin: Darwin
-      linux: Linux
-      windows: Windows
-      386: i386
-      amd64: x86_64
+  - name_template: >-
+      {{ .ProjectName }}_
+      {{- .Version }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
 changelog:
   sort: asc
   use: github

Makefile

@@ -5,8 +5,13 @@ VERSION=$(shell git describe --tags --dirty --always)
 build:
 	go build -ldflags "-X 'github.com/conduitio-labs/conduit-connector-hubspot.version=${VERSION}'" -o conduit-connector-hubspot cmd/connector/main.go
 
+# if you plan to run integration tests it's recommended to do that with -p=1 specified.
 test:
 	go test $(GOTEST_FLAGS) ./...
 
 lint:
 	golangci-lint run
+
+mockgen:
+	mockgen -package mock -source source/source.go -destination source/mock/source.go
+	mockgen -package mock -source destination/destination.go -destination destination/mock/destination.go

README.md

@@ -2,4 +2,94 @@
 
 ## General
 
-The [HubSpot](https://hubspot.com/) connector is one of Conduit plugins. It provides both, a source and a destination HubSpot connector.
+The [HubSpot](https://hubspot.com/) connector is one of Conduit plugins. It provides both, a source and a destination HubSpot connector.
+
+### Prerequisites
+
+- [Go](https://go.dev/) v1.20
+- [HubSpot](https://www.hubspot.com/) account and a [private app](https://developers.hubspot.com/docs/api/private-apps)
+- (optional) [golangci-lint](https://github.com/golangci/golangci-lint) v1.49.0
+
+### How to build it
+
+Run `make build`.
+
+### Testing
+
+Run `make test` to run all the unit and integration tests. The integration tests require the environment variable `HUBSPOT_ACCESS_TOKEN` to be set. If the environment variable is empty, the integration tests will be skipped.
+
+## Source
+
+The HubSpot Source Connector uses a private app access token to connect to a HubSpot account and creates records for each resource change detected in a HubSpot account.
+
+### Snapshot capture
+
+When the connector first starts, snapshot mode is enabled. The connector reads items of a resource that you specified. It only reads items that are created before the connector starts to run, batching them by `bufferSize`. Each new batch is processed every `pollingPeriod` duration. Once all items in that initial snapshot are read the connector switches into CDC mode.
+
+This behavior is enabled by default, but can be turned off by adding `"snapshot": false` to the Source configuration.
+
+### Change Data Capture
+
+When a snapshot is captured the connector starts to listen to data changes. It can track creates, updates, and deletes that occur after the connector is started. But please note that not all resources support all operations. You can check the available resources and operations they support out [here](docs/resources.md).
+
+### Position structure
+
+The connector goes through two modes.
+
+**Snapshot**. The position contains the `initialTimestamp` field that is equal to the timestamp of the first connector run. If a resource supports filtering by id, the position also contains the id of the last processed item in the `itemId` field.
+
+Here's an example of a Snapshot position:
+
+```json
+{
+  "mode": "snapshot",
+  "itemId": "256",
+  "initialTimestamp": "2022-10-28T14:58:27Z"
+}
+```
+
+**CDC**. The position in this mode contains the same fields as in the Snapshot mode plus a `timestamp` that is equal to the `updatedAt` of the last processed item.
+
+Here's an example of a CDC position:
+
+```json
+{
+  "mode": "cdc",
+  "itemId": "256",
+  "initialTimestamp": "2022-10-28T14:58:27Z",
+  "timestamp": "2022-10-28T15:00:50Z"
+}
+```
+
+### Configuration options
+
+| name              | description                                                                                                                                                                                                                                                                                               | required | default |
+| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- |
+| `accessToken`     | The private app access token for accessing the HubSpot API.                                                                                                                                                                                                                                               | **true** |         |
+| `resource`        | The HubSpot resource that the connector will work with.<br />You can find a list of the available resources [here](docs/resources.md).                                                                                                                                                                    | **true** |         |
+| `maxRetries`      | The number of HubSpot API request retries attempts that will be tried before giving up if a request fails.                                                                                                                                                                                                | false    | `4`     |
+| `pollingPeriod`   | The duration that defines a period of polling new items.                                                                                                                                                                                                                                                  | false    | `5s`    |
+| `bufferSize`      | The buffer size for consumed items.<br />It will also be used as a limit when retrieving items from the HubSpot API.                                                                                                                                                                                      | false    | `100`   |
+| `extraProperties` | The list of HubSpot resource properties to include in addition to the default.<br />If any of the specified properties are not present on the requested HubSpot resource, they will be ignored.<br />Only CRM resources support this.<br />The format of this field is the following: `prop1,prop2,prop3` | false    |         |
+| `snapshot`        | The field determines whether or not the connector will take a snapshot of the entire collection before starting CDC mode.                                                                                                                                                                                 | false    | `true`  |
+
+### Known limitations
+
+- Not all resources support all CDC operations. You can check the available resources and operations they support out [here](docs/resources.md).
+
+## Destination
+
+The HubSpot Destination takes a `record.Record` and sends its payload to HubSpot without any transformations. The destination is designed to handle different payloads. You can check the available resources and operations they support out [here](/docs/resources.md).
+
+### Configuration options
+
+| name            | description                                                                                                                            | required | default |
+| --------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- |
+| `accessToken`   | The private app access token for accessing the HubSpot API.                                                                            | **true** |         |
+| `resource`      | The HubSpot resource that the connector will work with.<br />You can find a list of the available resources [here](docs/resources.md). | **true** |         |
+| `maxRetries`    | The number of HubSpot API request retries attempts that will be tried before giving up if a request fails.                             | false    | `4`     |
+
+### Known limitations
+
+- To perform the update or delete operations the destination requires the `record.Key` to be set.
+- If you want to use the destination to insert records from a source containing records that were previously taken by the HubSpot source, you may need to exclude some read-only fields from their payload. When trying to insert read-only fields you'll see an appropriate error message in the logs containing the names of the read-only fields.