Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Add Cloud deploy functionality out of experimental status #419

Merged
merged 33 commits into from
Dec 10, 2024
Merged
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
9b40875
clean up cloud functionality
aaronsteers Oct 15, 2024
4777d24
fix docs generation
aaronsteers Oct 15, 2024
6799b29
undo test skip for Cloud integration tests
aaronsteers Dec 6, 2024
1d4f53e
Merge remote-tracking branch 'origin/main' into aj/feat/cloud-deploy-…
aaronsteers Dec 6, 2024
1664110
Merge remote-tracking branch 'origin/main' into aj/feat/cloud-deploy-…
aaronsteers Dec 7, 2024
e8e948d
some name fixes
aaronsteers Dec 7, 2024
b435550
misc updates (not running tests yet)
aaronsteers Dec 7, 2024
354548e
more fixes
aaronsteers Dec 7, 2024
fe0ce9c
add: _paired_destination_name
aaronsteers Dec 7, 2024
be9225d
rename to 'cache_type_to_destination_type'
aaronsteers Dec 7, 2024
d8b40d1
fix more tests
aaronsteers Dec 7, 2024
7c9a6ac
multiple fixes
aaronsteers Dec 9, 2024
d58bf93
improve tests
aaronsteers Dec 9, 2024
e5b6ec4
xfail postgres merge support check
aaronsteers Dec 9, 2024
413abf4
update tests
aaronsteers Dec 9, 2024
3e16ee4
fix more tests
aaronsteers Dec 9, 2024
9d52230
improve tests
aaronsteers Dec 9, 2024
90bf1a2
lint fixes
aaronsteers Dec 9, 2024
2ab2e68
lint fixes
aaronsteers Dec 9, 2024
8a1092e
ignore warnings
aaronsteers Dec 9, 2024
c8a41b5
chore: convert if/then to lookup mapping
aaronsteers Dec 10, 2024
452bc0e
fixes (full code review)
aaronsteers Dec 10, 2024
13a4656
apply fixes and suggestions
aaronsteers Dec 10, 2024
e4b010f
delete redundant test
aaronsteers Dec 10, 2024
b8d63f0
fixes and apply suggestions
aaronsteers Dec 10, 2024
349df46
apply suggestion (class var)
aaronsteers Dec 10, 2024
6c3127e
improve exception handling
aaronsteers Dec 10, 2024
6325a53
Merge remote-tracking branch 'origin/main' into aj/feat/cloud-deploy-…
aaronsteers Dec 10, 2024
1bdcab1
fix list_destinations
aaronsteers Dec 10, 2024
ec5398b
fix return type
aaronsteers Dec 10, 2024
e4397d4
import types from api package
aaronsteers Dec 10, 2024
34f925e
Auto-fix lint and format issues
Dec 10, 2024
93fb13c
Auto-fix lint issues (unsafe)
Dec 10, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airbyte/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@

# Submodules imported here for documentation reasons: https://github.com/mitmproxy/pdoc/issues/757
if TYPE_CHECKING:
# ruff: noqa: TCH004 # imports used for more than type checking
# ruff: noqa: TC004 # imports used for more than type checking
from airbyte import (
caches,
callbacks,
Expand Down
7 changes: 5 additions & 2 deletions airbyte/_executors/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@

from airbyte import exceptions as exc
from airbyte._message_iterators import AirbyteMessageIterator
from airbyte.sources.registry import ConnectorMetadata


if TYPE_CHECKING:
from collections.abc import Generator, Iterable, Iterator

from airbyte.sources.registry import ConnectorMetadata


_LATEST_VERSION = "latest"

Expand Down Expand Up @@ -161,7 +162,9 @@ def __init__(
if not name and not metadata:
raise exc.PyAirbyteInternalError(message="Either name or metadata must be provided.")

self.name: str = name or cast(ConnectorMetadata, metadata).name # metadata is not None here
self.name: str = (
name or cast("ConnectorMetadata", metadata).name
) # metadata is not None here
self.metadata: ConnectorMetadata | None = metadata
self.enforce_version: bool = target_version is not None

Expand Down
2 changes: 1 addition & 1 deletion airbyte/_executors/declarative.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def __init__(
self.name = name
self._manifest_dict: dict
if isinstance(manifest, Path):
self._manifest_dict = cast(dict, yaml.safe_load(manifest.read_text()))
self._manifest_dict = cast("dict", yaml.safe_load(manifest.read_text()))

elif isinstance(manifest, dict):
self._manifest_dict = manifest
Expand Down
2 changes: 1 addition & 1 deletion airbyte/_executors/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def _try_get_source_manifest(
)
response.raise_for_status() # Raise HTTPError exception if the download failed
try:
return cast(dict, yaml.safe_load(response.text))
return cast("dict", yaml.safe_load(response.text))
except yaml.YAMLError as ex:
raise exc.AirbyteConnectorInstallationError(
message="Failed to parse the connector manifest YAML.",
Expand Down
10 changes: 5 additions & 5 deletions airbyte/_message_iterators.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

from __future__ import annotations

import datetime
import sys
from collections.abc import Iterator
from typing import IO, TYPE_CHECKING, cast
Expand All @@ -27,6 +26,7 @@


if TYPE_CHECKING:
import datetime
from collections.abc import Callable, Generator, Iterable, Iterator
from pathlib import Path

Expand Down Expand Up @@ -98,7 +98,7 @@ def generator() -> Generator[AirbyteMessage, None, None]:
data=record,
emitted_at=int(
cast(
datetime.datetime, record.get(AB_EXTRACTED_AT_COLUMN)
"datetime.datetime", record.get(AB_EXTRACTED_AT_COLUMN)
).timestamp()
),
# `meta` and `namespace` are not handled:
Expand Down Expand Up @@ -134,7 +134,7 @@ def generator() -> Generator[AirbyteMessage, None, None]:
yield AirbyteMessage.model_validate_json(next_line)
except pydantic.ValidationError:
# Handle JSON decoding errors (optional)
raise ValueError("Invalid JSON format") # noqa: B904, TRY003
raise ValueError("Invalid JSON format") # noqa: B904

return cls(generator())

Expand All @@ -149,7 +149,7 @@ def generator() -> Generator[AirbyteMessage, None, None]:
yield AirbyteMessage.model_validate_json(line)
except pydantic.ValidationError:
# Handle JSON decoding errors (optional)
raise ValueError(f"Invalid JSON format in input string: {line}") # noqa: B904, TRY003
raise ValueError(f"Invalid JSON format in input string: {line}") # noqa: B904

return cls(generator())

Expand Down Expand Up @@ -193,6 +193,6 @@ def generator() -> Generator[AirbyteMessage, None, None]:
# Handle JSON decoding errors
current_file_buffer.close()
current_file_buffer = None
raise ValueError("Invalid JSON format") # noqa: B904, TRY003
raise ValueError("Invalid JSON format") # noqa: B904

return cls(generator())
2 changes: 1 addition & 1 deletion airbyte/_processors/sql/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ class BigQueryTypeConverter(SQLTypeConverter):
@classmethod
def get_string_type(cls) -> sqlalchemy.types.TypeEngine:
"""Return the string type for BigQuery."""
return cast(sqlalchemy.types.TypeEngine, "String") # BigQuery uses STRING for all strings
return cast("sqlalchemy.types.TypeEngine", "String") # BigQuery uses STRING for all strings

@overrides
def to_sql_type(
Expand Down
Loading
Loading