From 819a8aaef4b6b094748db3b3c0fd6f9ec817bfce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Eriksson?= Date: Wed, 20 Nov 2024 07:33:53 +0100 Subject: [PATCH] Support bucket refs in Encore.ts --- docs/ts/primitives/object-storage.md | 46 +++++++ .../js/encore.dev/storage/objects/bucket.ts | 8 +- runtimes/js/encore.dev/storage/objects/mod.ts | 1 + .../js/encore.dev/storage/objects/refs.ts | 33 +++++ tsparser/src/legacymeta/mod.rs | 6 +- tsparser/src/parser/parser.rs | 5 +- .../src/parser/resources/infra/objects.rs | 122 +++++++++++++++++- tsparser/src/parser/types/object.rs | 4 +- tsparser/src/parser/types/typ.rs | 9 +- tsparser/src/parser/types/type_resolve.rs | 15 ++- tsparser/src/parser/usageparser/mod.rs | 21 ++- v2/parser/infra/objects/errors.go | 2 +- v2/parser/infra/objects/usage.go | 48 ++++--- v2/parser/infra/objects/usage_test.go | 31 +++++ 14 files changed, 310 insertions(+), 41 deletions(-) create mode 100644 runtimes/js/encore.dev/storage/objects/refs.ts diff --git a/docs/ts/primitives/object-storage.md b/docs/ts/primitives/object-storage.md index c332c363e8..4ff068994a 100644 --- a/docs/ts/primitives/object-storage.md +++ b/docs/ts/primitives/object-storage.md @@ -123,3 +123,49 @@ If an upload fails due to a precondition not being met (like if the object alrea and the `notExists: true` option is set), it throws a `PreconditionFailed` error. Other errors are returned as `ObjectsError` errors (which the above errors also extend). + +## Bucket references + +Encore uses static analysis to determine which services are accessing each bucket, +and what operations each service is performing. + +That information is used for features such as rendering architecture diagrams, and can be used by Encore's Cloud Platform to provision infrastructure correctly and configure IAM permissions. + +This means `Bucket` objects can't be passed around however you like, +as it makes static analysis impossible in many cases. To simplify your workflow, given these restrictions, +Encore supports defining a "reference" to a bucket that can be passed around any way you want. + +### Using bucket references + +Define a bucket reference by calling `bucket.ref()` from within a service, where `DesiredPermissions` is one of the pre-defined permission types defined in the `encore.dev/storage/objects` module. + +This means you're effectively pre-declaring the permissions you need, and only the methods that +are allowed by those permissions are available on the returned reference object. + +For example, to get a reference to a bucket that can only download objects: + +```typescript +import { Uploader } from "encore.dev/storage/objects"; +const ref = profilePictures.ref(); + +// You can now freely pass around `ref`, and you can use +// `ref.upload()` just like you would `profilePictures.upload()`. +``` + +To ensure Encore still is aware of which permissions each service needs, the call to `bucket.ref` +must be made from within a service, so that Encore knows which service to associate the permissions with. + +Encore provides permission interfaces for each operation that can be performed on a bucket: + +* `Downloader` for downloading objects +* `Uploader` for uploading objects +* `Lister` for listing objects +* `Attrser` for getting object attributes +* `Remover` for removing objects + +If you need multiple permissions you can combine them using `&`. +For example, `profilePictures.ref` gives you a reference +that allows calling both `download` and `upload`. + +For convenience Encore also provides a `ReadWriter` permission that gives complete read-write access +to the bucket, granting all the permissions above. It is equivalent to `Downloader & Uploader & Lister & Attrser & Remover`. diff --git a/runtimes/js/encore.dev/storage/objects/bucket.ts b/runtimes/js/encore.dev/storage/objects/bucket.ts index aef9cd44fd..7d9066cb37 100644 --- a/runtimes/js/encore.dev/storage/objects/bucket.ts +++ b/runtimes/js/encore.dev/storage/objects/bucket.ts @@ -2,6 +2,7 @@ import { getCurrentRequest } from "../../internal/reqtrack/mod"; import * as runtime from "../../internal/runtime/mod"; import { StringLiteral } from "../../internal/utils/constraints"; import { unwrapErr } from "./error"; +import { BucketPerms, Uploader, Downloader, Attrser, Lister, Remover } from "./refs"; export interface BucketConfig { /** @@ -14,7 +15,7 @@ export interface BucketConfig { /** * Defines a new Object Storage bucket infrastructure resource. */ -export class Bucket { +export class Bucket extends BucketPerms implements Uploader, Downloader, Attrser, Lister, Remover { impl: runtime.Bucket; /** @@ -22,6 +23,7 @@ export class Bucket { */ // eslint-disable-next-line @typescript-eslint/no-unused-vars constructor(name: string, cfg?: BucketConfig) { + super(); this.impl = runtime.RT.bucket(name); } @@ -100,6 +102,10 @@ export class Bucket { unwrapErr(err); } } + + ref

(): P { + return this as unknown as P + } } export interface ListOptions { diff --git a/runtimes/js/encore.dev/storage/objects/mod.ts b/runtimes/js/encore.dev/storage/objects/mod.ts index 7b5c002230..c8fdb82e62 100644 --- a/runtimes/js/encore.dev/storage/objects/mod.ts +++ b/runtimes/js/encore.dev/storage/objects/mod.ts @@ -1,3 +1,4 @@ export { Bucket } from "./bucket"; export type { BucketConfig, ObjectAttrs, UploadOptions } from "./bucket"; export { ObjectsError, ObjectNotFound, PreconditionFailed } from "./error"; +export type { BucketPerms, Uploader, Downloader, Attrser, Lister, ReadWriter } from "./refs"; diff --git a/runtimes/js/encore.dev/storage/objects/refs.ts b/runtimes/js/encore.dev/storage/objects/refs.ts new file mode 100644 index 0000000000..a6dbe83e03 --- /dev/null +++ b/runtimes/js/encore.dev/storage/objects/refs.ts @@ -0,0 +1,33 @@ +import type { AttrsOptions, DeleteOptions, DownloadOptions, ExistsOptions, ListEntry, ListOptions, ObjectAttrs, UploadOptions } from "./bucket"; + +export abstract class BucketPerms { + private bucketPerms(): void { }; +} + +export abstract class Uploader extends BucketPerms { + abstract upload(name: string, data: Buffer, options?: UploadOptions): Promise; +} + +export abstract class Downloader extends BucketPerms { + abstract download(name: string, options?: DownloadOptions): Promise; +} + +export abstract class Attrser extends BucketPerms { + abstract attrs(name: string, options?: AttrsOptions): Promise; + abstract exists(name: string, options?: ExistsOptions): Promise; +} + +export abstract class Lister extends BucketPerms { + abstract list(options: ListOptions): AsyncGenerator; +} + +export abstract class Remover extends BucketPerms { + abstract remove(name: string, options?: DeleteOptions): Promise; +} + +export type ReadWriter = + & Uploader + & Downloader + & Attrser + & Lister + & Remover; diff --git a/tsparser/src/legacymeta/mod.rs b/tsparser/src/legacymeta/mod.rs index bf6cfb2aa9..1be8aa880b 100644 --- a/tsparser/src/legacymeta/mod.rs +++ b/tsparser/src/legacymeta/mod.rs @@ -388,7 +388,7 @@ impl<'a> MetaBuilder<'a> { }; use objects::Operation; - let op = match access.op { + let ops = access.ops.iter().map(|op| match op { Operation::DeleteObject => v1::bucket_usage::Operation::DeleteObject, Operation::ListObjects => v1::bucket_usage::Operation::ListObjects, Operation::ReadObjectContents => { @@ -401,7 +401,7 @@ impl<'a> MetaBuilder<'a> { Operation::GetObjectMetadata => { v1::bucket_usage::Operation::GetObjectMetadata } - }; + } as i32); let idx = svc_index.get(&svc.name).unwrap(); bucket_perms @@ -411,7 +411,7 @@ impl<'a> MetaBuilder<'a> { operations: vec![], }) .operations - .push(op as i32); + .extend(ops); } Usage::CallEndpoint(call) => { diff --git a/tsparser/src/parser/parser.rs b/tsparser/src/parser/parser.rs index 3a2afeb7c5..7322812702 100644 --- a/tsparser/src/parser/parser.rs +++ b/tsparser/src/parser/parser.rs @@ -255,12 +255,13 @@ impl<'a> Parser<'a> { resources.extend(additional_resources); binds.extend(additional_binds); - let resolver = UsageResolver::new(&self.pc.loader, &resources, &binds); + let resolver = + UsageResolver::new(&self.pc.loader, &self.pc.type_checker, &resources, &binds); let mut usages = Vec::new(); for module in self.pc.loader.modules() { let exprs = resolver.scan_usage_exprs(&module); - let u = resolver.resolve_usage(&exprs)?; + let u = resolver.resolve_usage(&module, &exprs)?; usages.extend(u); } diff --git a/tsparser/src/parser/resources/infra/objects.rs b/tsparser/src/parser/resources/infra/objects.rs index 1df8b2bffe..95d35a3916 100644 --- a/tsparser/src/parser/resources/infra/objects.rs +++ b/tsparser/src/parser/resources/infra/objects.rs @@ -1,3 +1,5 @@ +use std::ops::Deref; + use anyhow::Result; use litparser::LitParser; use litparser_derive::LitParser; @@ -12,7 +14,8 @@ use crate::parser::resources::parseutil::{iter_references, TrackedNames}; use crate::parser::resources::parseutil::{NamedClassResourceOptionalConfig, NamedStaticMethod}; use crate::parser::resources::Resource; use crate::parser::resources::ResourcePath; -use crate::parser::usageparser::{ResolveUsageData, Usage, UsageExprKind}; +use crate::parser::types::{Generic, Type}; +use crate::parser::usageparser::{MethodCall, ResolveUsageData, Usage, UsageExprKind}; use crate::parser::Range; use crate::span_err::ErrReporter; @@ -94,6 +97,24 @@ pub const OBJECTS_PARSER: ResourceParser = ResourceParser { pub fn resolve_bucket_usage(data: &ResolveUsageData, bucket: Lrc) -> Result> { Ok(match &data.expr.kind { UsageExprKind::MethodCall(call) => { + if call.method.as_ref() == "ref" { + let Some(type_args) = call.call.type_args.as_deref() else { + call.call + .span + .err("expected a type argument in call to Bucket.ref"); + return Ok(None); + }; + + let Some(type_arg) = type_args.params.first() else { + call.call + .span + .err("expected a type argument in call to Bucket.ref"); + return Ok(None); + }; + + return parse_bucket_ref(data, bucket, call, type_arg); + } + let op = match call.method.as_ref() { "list" => Operation::ListObjects, "exists" | "attrs" => Operation::GetObjectMetadata, @@ -109,7 +130,7 @@ pub fn resolve_bucket_usage(data: &ResolveUsageData, bucket: Lrc) -> Res Some(Usage::Bucket(BucketUsage { range: data.expr.range, bucket, - op, + ops: vec![op], })) } @@ -123,11 +144,106 @@ pub fn resolve_bucket_usage(data: &ResolveUsageData, bucket: Lrc) -> Res }) } +fn parse_bucket_ref( + data: &ResolveUsageData, + bucket: Lrc, + _call: &MethodCall, + type_arg: &ast::TsType, +) -> Result> { + fn process_type( + data: &ResolveUsageData, + sp: &swc_common::Span, + t: &Type, + depth: usize, + ) -> Option> { + if depth > 10 { + // Prevent infinite recursion. + return None; + } + + match t { + Type::Named(named) => { + let ops = match named.obj.name.as_deref() { + Some("Lister") => vec![Operation::ListObjects], + Some("Attrser") => vec![Operation::GetObjectMetadata], + Some("Uploader") => vec![Operation::WriteObject], + Some("Downloader") => vec![Operation::ReadObjectContents], + Some("Remover") => vec![Operation::DeleteObject], + _ => { + let underlying = data.type_checker.resolve_obj_type(&named.obj); + return process_type(data, sp, &underlying, depth + 1); + } + }; + + Some(ops) + } + + Type::Class(cls) => { + let ops = cls + .methods + .iter() + .filter_map(|method| { + let op = match method.as_str() { + "list" => Operation::ListObjects, + "exists" | "attrs" => Operation::GetObjectMetadata, + "upload" => Operation::WriteObject, + "download" => Operation::ReadObjectContents, + "remove" => Operation::DeleteObject, + _ => { + // Ignore other methods. + return None; + } + }; + + Some(op) + }) + .collect(); + Some(ops) + } + + Type::Generic(Generic::Intersection(int)) => { + let mut result = Vec::new(); + for t in &[&int.x, &int.y] { + if let Some(ops) = process_type(data, sp, t, depth + 1) { + result.extend(ops); + } + } + + if result.is_empty() { + None + } else { + Some(result) + } + } + + _ => { + sp.err(&format!("unsupported bucket permission type {:#?}", t)); + None + } + } + } + + let typ = data + .type_checker + .resolve_type(data.module.clone(), type_arg); + + if let Some(ops) = process_type(data, &typ.span(), typ.deref(), 0) { + Ok(Some(Usage::Bucket(BucketUsage { + range: data.expr.range, + bucket, + ops, + }))) + } else { + typ.err("no bucket permissions found in type argument"); + Ok(None) + } +} + #[derive(Debug)] pub struct BucketUsage { pub range: Range, pub bucket: Lrc, - pub op: Operation, + pub ops: Vec, } #[derive(Debug)] diff --git a/tsparser/src/parser/types/object.rs b/tsparser/src/parser/types/object.rs index 2f8e456702..84940a9b30 100644 --- a/tsparser/src/parser/types/object.rs +++ b/tsparser/src/parser/types/object.rs @@ -111,7 +111,7 @@ pub enum TypeNameDecl { #[derive(Debug)] pub struct Class { #[allow(dead_code)] - spec: Box, + pub spec: Box, } #[derive(Debug)] @@ -679,7 +679,7 @@ impl ResolveState { }) } - pub(super) fn lookup_module(&self, id: ModuleId) -> Option> { + pub fn lookup_module(&self, id: ModuleId) -> Option> { self.modules.borrow().get(&id).cloned() } diff --git a/tsparser/src/parser/types/typ.rs b/tsparser/src/parser/types/typ.rs index 5fecb2b918..0e9e2bd072 100644 --- a/tsparser/src/parser/types/typ.rs +++ b/tsparser/src/parser/types/typ.rs @@ -277,6 +277,7 @@ impl InterfaceField { #[derive(Debug, Clone, Hash, Serialize)] pub struct ClassType { + pub methods: Vec, // TODO: include class fields here } @@ -826,10 +827,10 @@ pub fn intersect<'a: 'b, 'b>( } (Type::Class(_), Type::Class(_)) => { - HANDLER.with(|handler| { - handler.err("intersection of class types is not yet supported"); - }); - Cow::Owned(Type::Basic(Basic::Never)) + Cow::Owned(Type::Generic(Generic::Intersection(Intersection { + x: Box::new(a.into_owned()), + y: Box::new(b.into_owned()), + }))) } (Type::Named(x), _) => { diff --git a/tsparser/src/parser/types/type_resolve.rs b/tsparser/src/parser/types/type_resolve.rs index 4638b0c913..f4a95106f9 100644 --- a/tsparser/src/parser/types/type_resolve.rs +++ b/tsparser/src/parser/types/type_resolve.rs @@ -1381,7 +1381,20 @@ impl<'a> Ctx<'a> { Type::Basic(Basic::Never) } - ObjectKind::Class(_o) => Type::Class(ClassType {}), + ObjectKind::Class(o) => { + let methods = o + .spec + .body + .iter() + .filter_map(|mem| match mem { + ast::ClassMember::Method(m) => { + m.key.as_ident().map(|id| id.sym.to_string()) + } + _ => None, + }) + .collect(); + Type::Class(ClassType { methods }) + } ObjectKind::Module(_o) => Type::Basic(Basic::Never), ObjectKind::Namespace(_o) => { diff --git a/tsparser/src/parser/usageparser/mod.rs b/tsparser/src/parser/usageparser/mod.rs index 4a03630a53..e5597ca6ca 100644 --- a/tsparser/src/parser/usageparser/mod.rs +++ b/tsparser/src/parser/usageparser/mod.rs @@ -13,6 +13,8 @@ use crate::parser::resourceparser::bind::Bind; use crate::parser::resources::{apis, infra, Resource}; use crate::parser::Range; +use super::types::TypeChecker; + #[derive(Debug)] pub struct UsageExpr { pub range: Range, @@ -44,7 +46,7 @@ pub enum UsageExprKind { #[derive(Debug)] pub struct MethodCall { pub method: ast::Ident, - _call: ast::CallExpr, + pub call: ast::CallExpr, } #[derive(Debug)] @@ -76,6 +78,7 @@ pub struct Other { pub struct UsageResolver<'a> { module_loader: &'a ModuleLoader, + type_checker: &'a TypeChecker, resources: &'a [Resource], binds_by_module: HashMap>>, } @@ -83,11 +86,13 @@ pub struct UsageResolver<'a> { impl<'a> UsageResolver<'a> { pub fn new( module_loader: &'a ModuleLoader, + type_checker: &'a TypeChecker, resources: &'a [Resource], binds: &[Lrc], ) -> Self { let mut resolver = Self { module_loader, + type_checker, resources, binds_by_module: HashMap::new(), }; @@ -221,17 +226,21 @@ pub enum Usage { } pub struct ResolveUsageData<'a> { + pub module: &'a Lrc, + pub type_checker: &'a TypeChecker, pub expr: &'a UsageExpr, pub resources: &'a [Resource], } impl UsageResolver<'_> { - pub fn resolve_usage(&self, exprs: &[UsageExpr]) -> Result> { + pub fn resolve_usage(&self, module: &Lrc, exprs: &[UsageExpr]) -> Result> { let mut usages = Vec::new(); for expr in exprs { let data = ResolveUsageData { - resources: self.resources, + module, + type_checker: self.type_checker, expr, + resources: self.resources, }; match &expr.bind.resource { Resource::APIEndpoint(ep) => { @@ -355,7 +364,7 @@ impl<'a> UsageVisitor<'a> { range: call.span.into(), bind: bind.clone(), kind: UsageExprKind::MethodCall(MethodCall { - _call: (*call).to_owned(), + call: (*call).to_owned(), method: id.to_owned(), }), }) @@ -550,7 +559,7 @@ export const Bar = 5; })]; let resources = [res]; - let ur = UsageResolver::new(&pc.loader, &resources, &bar_binds); + let ur = UsageResolver::new(&pc.loader, &pc.type_checker, &resources, &bar_binds); let result = ur.external_binds_to_scan_for(foo_mod); assert_eq!(result.len(), 1); @@ -636,7 +645,7 @@ export const Bar = 5; })]; let resources = [res]; - let ur = UsageResolver::new(&pc.loader, &resources, &bar_binds); + let ur = UsageResolver::new(&pc.loader, &pc.type_checker, &resources, &bar_binds); let usages = ur.scan_usage_exprs(foo_mod); assert_eq!(usages.len(), 6); diff --git a/v2/parser/infra/objects/errors.go b/v2/parser/infra/objects/errors.go index dc8dc7bfc5..93059eba81 100644 --- a/v2/parser/infra/objects/errors.go +++ b/v2/parser/infra/objects/errors.go @@ -47,7 +47,7 @@ var ( errBucketRefInvalidPerms = errRange.New( "Unrecognized permissions in call to objects.BucketRef", - "The supported permissions are objects.Uploader/Downloader.", + "The supported permissions are objects.{Uploader,Downloader,Attrser,Lister,Remover}.", ) ErrBucketRefOutsideService = errRange.New( diff --git a/v2/parser/infra/objects/usage.go b/v2/parser/infra/objects/usage.go index d51bfa8c81..f16725939c 100644 --- a/v2/parser/infra/objects/usage.go +++ b/v2/parser/infra/objects/usage.go @@ -1,6 +1,8 @@ package objects import ( + "slices" + "encr.dev/pkg/option" "encr.dev/v2/internals/perr" "encr.dev/v2/internals/pkginfo" @@ -105,25 +107,35 @@ func parseBucketRef(errs *perr.List, expr *usage.FuncArg) usage.Usage { return nil } - checkUsage := func(typ schema.Type) (usage.Usage, bool) { - var perms []Perm - switch { - case isNamed(typ, "Uploader"): - perms = []Perm{WriteObject} - case isNamed(typ, "Downloader"): - perms = []Perm{ReadObjectContents} - case isNamed(typ, "Lister"): - perms = []Perm{ListObjects} - case isNamed(typ, "Remover"): - perms = []Perm{DeleteObject} - case isNamed(typ, "Attrser"): - perms = []Perm{GetObjectMetadata} - case isNamed(typ, "ReadWriter"): - perms = []Perm{WriteObject, ReadObjectContents, ListObjects, DeleteObject, GetObjectMetadata, UpdateObjectMetadata} - default: + checkUsage := func(types ...schema.Type) (usage.Usage, bool) { + if len(types) == 0 { return nil, false } + var perms []Perm + for _, typ := range types { + switch { + case isNamed(typ, "Uploader"): + perms = append(perms, WriteObject) + case isNamed(typ, "Downloader"): + perms = append(perms, ReadObjectContents) + case isNamed(typ, "Lister"): + perms = append(perms, ListObjects) + case isNamed(typ, "Remover"): + perms = append(perms, DeleteObject) + case isNamed(typ, "Attrser"): + perms = append(perms, GetObjectMetadata) + case isNamed(typ, "ReadWriter"): + perms = append(perms, WriteObject, ReadObjectContents, ListObjects, DeleteObject, GetObjectMetadata) + default: + return nil, false + } + } + + // Sort and de-dup the perms. + slices.Sort(perms) + slices.Compact(perms) + return &RefUsage{ Base: usage.Base{ File: expr.File, @@ -151,8 +163,8 @@ func parseBucketRef(errs *perr.List, expr *usage.FuncArg) usage.Usage { // Otherwise make sure the interface only embeds the one supported type we have (pubsub.Publisher). // We'll need to extend this in the future to support multiple permissions. if iface, ok := underlying.(schema.InterfaceType); ok { - if len(iface.EmbeddedIfaces) == 1 && len(iface.Methods) == 0 && len(iface.TypeLists) == 0 { - if u, ok := checkUsage(iface.EmbeddedIfaces[0]); ok { + if len(iface.Methods) == 0 && len(iface.TypeLists) == 0 { + if u, ok := checkUsage(iface.EmbeddedIfaces...); ok { return u } } diff --git a/v2/parser/infra/objects/usage_test.go b/v2/parser/infra/objects/usage_test.go index e534f00c4a..5b36428276 100644 --- a/v2/parser/infra/objects/usage_test.go +++ b/v2/parser/infra/objects/usage_test.go @@ -39,6 +39,24 @@ var ref = objects.BucketRef[objects.Uploader](bkt) Perms: []objects.Perm{objects.WriteObject}, }}, }, + { + Name: "ref_multi", + Code: ` +var bkt = objects.NewBucket("bucket", objects.BucketConfig{}) + +var ref = objects.BucketRef[objects.ReadWriter](bkt) +`, + Want: []usage.Usage{&objects.RefUsage{ + Perms: []objects.Perm{ + objects.DeleteObject, + objects.GetObjectMetadata, + objects.ListObjects, + objects.ReadObjectContents, + objects.UpdateObjectMetadata, + objects.WriteObject, + }, + }}, + }, { Name: "custom_ref_alias", Code: ` @@ -65,6 +83,19 @@ var ref = objects.BucketRef[MyRef](bkt) Perms: []objects.Perm{objects.WriteObject}, }}, }, + { + Name: "custom_ref_interface_multi", + Code: ` +var bkt = objects.NewBucket("bucket", objects.BucketConfig{}) + +type MyRef interface { objects.Uploader; objects.Downloader } + +var ref = objects.BucketRef[MyRef](bkt) +`, + Want: []usage.Usage{&objects.RefUsage{ + Perms: []objects.Perm{objects.ReadObjectContents, objects.WriteObject}, + }}, + }, { Name: "invalid_ref", Code: `