Skip to content

Commit

Permalink
Add support for signed upload urls.
Browse files Browse the repository at this point in the history
  • Loading branch information
erikcarlsson committed Jan 14, 2025
1 parent 482d69f commit 0457245
Showing 26 changed files with 1,533 additions and 891 deletions.
46 changes: 46 additions & 0 deletions cli/daemon/run/runtime_config2.go
Original file line number Diff line number Diff line change
@@ -378,6 +378,11 @@ func (g *RuntimeConfigGenerator) initialize() error {
Gcs: &runtimev1.BucketCluster_GCS{
Endpoint: &bktProviderConfig.GCS.Endpoint,
Anonymous: true,
LocalSign: &runtimev1.BucketCluster_GCS_LocalSignOptions{
BaseUrl: publicBaseURL,
AccessId: "[email protected]",
PrivateKey: reverseString(dummyPrivateKeyReversed),
},
},
},
})
@@ -833,3 +838,44 @@ func gzipBytes(data []byte) []byte {
_ = w.Close()
return buf.Bytes()
}

func reverseString(s string) string {
runes := []rune(s)
for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
runes[i], runes[j] = runes[j], runes[i]
}
return string(runes)
}

// We lightly obfuscate the PK to trigger fewer of the tools that warn about
// keys in source code.
//
// $ tail -r pk.pem | rev
const dummyPrivateKeyReversed = `-----YEK ETAVIRP DNE-----
=AOz3eEM5xAe/71Tfx3sQNkW
4FXBCChkppSrCoQnR6pBeP31wu0S0UTTNDhNmSYcerdSFbRhyZOzNRnhF9o1h5D5
+gKkhRZkC33z5+0p8aWwOVWJY8MDycHwvEYvtwcXLNZBHI8L8++mhp0uFz5c5sNM
pPRyurcUY36iDzx7hAJcAGoAvXJwVzTmzXBZtvFPs6Alc5gHti2W1l2bz2mwOV77
BA9xAW4R6EHVTnqaoXvxvocW5Z9I0ecJzx0NPfkXBriW1lNclAnkoRAYqziasa6C
WIxePQ2VRFbnLu7XR1M/xqg00GHFV0fTlNPo95lC6tl0PAdoupOX1lwjH3rQnTkB
Y4BgBKQQJ8F0PPTSMAvyK1bcHP2Iob8UFxyHuPOm11aHYwM4VZvmHm8jX/8vz4eb
6kbNbEkWzfJbbEen/EJLR1XtzvTdjs9bQnJvhQMZmPGzQalqHcVuilQX+PFV4ezM
A23w1HCIq6vZqXLO8rXhe8S5hImwVSAKq6TK5dlYPOTIBp66lCQgBKwjkcQcX7tq
mr44FuVB7hqBMfnCB0kKcs1SuYgmfUQE41JGInsqjdpaFOwzQi4Jcx7TK44p9vn2
ik6i/hN7JSVA8kMImWIxtL18uVC/Rg0RpM2vcjd+pfgUDifZ1FVYCiL3WlEzDBlZ
bSmYdd57T70mEEiuV8QmGiIRrk6kZAMP4CQgBKQ4mIYJX2RJQ1j0V+iXwY/bg+N5
DPEWLB0w6ReZapNy4DSEMD1zm6IWUuo3rGfCsSKUD0xFR/YkauO5Q+GI2gKvmj5V
MRiysBL/8PCBwKiFKo1MFjCUfbV/ks49/OJYSOi9WIJiXEg5Tm56BDTH6I8rNdU1
lGIimbKIuzEBWUHsyDQgBKQQ8O/PDCI/SJSPYjkxw1fpX022hUvVW9pvtmd6v0vX
M5kMBkT60IwTWhF0DoAx4Uyn4rlPiJy5TUwjC0po/aCRV+ug5C+wIRTCtVCpqRyz
GeB4U/3WXHmSulzK5Dw4ADfbWSP0dAbNNOaFI4y6u+acEl5MFt3GN/jieITLsZNK
X18B7zHj7LR2f5k3xiJJ/7uNFl8SCcnVquvEI1qslUSTLEPCNoiy5iX/VVTmVNwv
dUi92s5oFMyJOFW5joggeeQ55BN6EsjQTnj/XetnpPe5wf5vvptHg5HOcUjJPmIJ
vsGpMXoyCh3mzdQPMUJM9Ha8DKlACadqTjdid9ZsAAYLAEggCEAABMgAvulUiO2B
FkdtezbN/f5vpPbr4knO22xylfkUp5Uw0W/HxtntXXobF42guEEiie49zki5fPHK
vAMC7bOERRLV4v35Dd9QV/KFe0FxqEfm8bFDM6FoA4c0qnkDaKbMhdvxxs0wVFRm
BukfBCLOt+W/XyFhZvUKkxgbcOjXV7HRFQGI+GZnrf00qbCRNOCdlYLoYX1kf3pQ
eNY6o9ZCJxIDO+dUATCoP3tmP4hvonrjGfpek99D4Ye3+iDwg0AxDW+bt9qoRFew
VdOuGmooPaDDxn95q5IghRhrvrEaHpkN/EZiNEAJWQkZa9wkxGye5T9hMZRBjUkt
wGPTyf02fuGquCQABIoAAEgAjSggwcKBCSAAFEQAB0w9GikhqkgBNADABIQvEIIM
-----YEK ETAVIRP NIGEB-----`
7 changes: 7 additions & 0 deletions pkg/rtconfgen/convert.go
Original file line number Diff line number Diff line change
@@ -411,6 +411,13 @@ func (c *legacyConverter) Convert() (*config.Runtime, error) {
Endpoint: prov.Gcs.GetEndpoint(),
Anonymous: prov.Gcs.Anonymous,
}
if opt := prov.Gcs.LocalSign; opt != nil {
p.GCS.LocalSign = &config.GCSLocalSignOptions{
BaseURL: opt.BaseUrl,
AccessID: opt.AccessId,
PrivateKey: opt.PrivateKey,
}
}
default:
c.setErrf("unknown object storage provider type %T", prov)
continue
967 changes: 487 additions & 480 deletions proto/encore/parser/meta/v1/meta.pb.go

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions proto/encore/parser/meta/v1/meta.proto
Original file line number Diff line number Diff line change
@@ -92,6 +92,10 @@ message BucketUsage {

// Get an bucket/object's public url.
GET_PUBLIC_URL = 7;

// Generating a signed URL to allow an external recipient to create or
// update an object.
SIGNED_UPLOAD_URL = 8;
}
}

873 changes: 488 additions & 385 deletions proto/encore/runtime/v1/infra.pb.go

Large diffs are not rendered by default.

14 changes: 14 additions & 0 deletions proto/encore/runtime/v1/infra.proto
Original file line number Diff line number Diff line change
@@ -340,6 +340,20 @@ message BucketCluster {

// Whether to connect anonymously or if a service account should be resolved.
bool anonymous = 2;

// Additional options for signed URLs when running in local dev mode.
// Only use with anonymous mode.
optional LocalSignOptions local_sign = 3;

message LocalSignOptions {
// Base prefix to use for presigned URLs.
string base_url = 1;

// Use these credentials to sign local URLs. Only pass dummy credentials
// here, no actual secrets.
string access_id = 2;
string private_key = 3;
}
}
}

1 change: 1 addition & 0 deletions runtimes/core/src/infracfg.rs
Original file line number Diff line number Diff line change
@@ -448,6 +448,7 @@ pub fn map_infra_to_runtime(infra: InfraConfig) -> RuntimeConfig {
pbruntime::bucket_cluster::Gcs {
endpoint: gcs.endpoint,
anonymous: false,
local_sign: None,
},
)),
buckets: gcs
99 changes: 98 additions & 1 deletion runtimes/core/src/objects/gcs/bucket.rs
Original file line number Diff line number Diff line change
@@ -3,16 +3,19 @@ use futures::TryStreamExt;
use google_cloud_storage::http::objects::download::Range;
use google_cloud_storage::http::objects::get::GetObjectRequest;
use google_cloud_storage::http::objects::upload::{Media, UploadObjectRequest, UploadType};
use google_cloud_storage::sign::SignBy;
use google_cloud_storage::sign::SignedURLOptions;
use std::borrow::Cow;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::time::SystemTime;
use tokio::io::AsyncRead;

use crate::encore::runtime::v1 as pb;
use crate::objects::{
AttrsOptions, DeleteOptions, DownloadOptions, DownloadStream, Error, ExistsOptions, ListEntry,
ListOptions, ObjectAttrs, PublicUrlError, UploadOptions,
ListOptions, ObjectAttrs, PublicUrlError, UploadOptions, UploadUrlOptions,
};
use crate::{objects, CloudName, EncoreName};
use google_cloud_storage as gcs;
@@ -26,16 +29,34 @@ pub struct Bucket {
cloud_name: CloudName,
public_base_url: Option<String>,
key_prefix: Option<String>,
local_sign: Option<LocalSignOptions>,
}

#[derive(Debug)]
pub struct LocalSignOptions {
base_url: String,
access_id: String,
private_key: String,
}

fn local_sign_config_from_client(client: &LazyGCSClient) -> Option<LocalSignOptions> {
client.cfg.local_sign.as_ref().map(|cfg| LocalSignOptions {
base_url: cfg.base_url.clone(),
access_id: cfg.access_id.clone(),
private_key: cfg.private_key.clone(),
})
}

impl Bucket {
pub(super) fn new(client: Arc<LazyGCSClient>, cfg: &pb::Bucket) -> Self {
let local_sign = local_sign_config_from_client(&client);
Self {
client,
encore_name: cfg.encore_name.clone().into(),
cloud_name: cfg.cloud_name.clone().into(),
public_base_url: cfg.public_base_url.clone(),
key_prefix: cfg.key_prefix.clone(),
local_sign,
}
}

@@ -207,6 +228,56 @@ impl objects::ObjectImpl for Object {
})
}

fn signed_upload_url(
self: Arc<Self>,
options: UploadUrlOptions,
) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>> {
Box::pin(async move {
match self.bkt.client.get().await {
Ok(client) => {
let gcs_opts = SignedURLOptions {
method: gcs::sign::SignedURLMethod::PUT,
expires: options.ttl,
start_time: Some(SystemTime::now()),
..Default::default()
};

// We use a fake GCS service for local development. Ideally, the runtime
// code would be oblivious to this once the GCS client is set up. But that
// turns out to be difficult for URL signing, so we add a special case
// here.
let local_sign = &self.bkt.local_sign;
let (access_id, sign_by) = match local_sign {
Some(opt) => (
Some(opt.access_id.clone()),
Some(SignBy::PrivateKey(opt.private_key.as_bytes().to_vec())),
),
None => (None, None),
};

let name = self.bkt.obj_name(Cow::Borrowed(&self.key)).into_owned();
let mut url = client
.signed_url(&self.bkt.cloud_name, &name, access_id, sign_by, gcs_opts)
.await
.map_err(|e| Error::Internal(e.into()))?;

// More special handling for the local dev case.
if let Some(cfg) = local_sign {
url = replace_url_prefix(&url, &cfg.base_url)
.into_owned()
.to_string();
}

Ok(url)
}
Err(err) => Err(Error::Internal(anyhow::anyhow!(
"unable to resolve client: {}",
err
))),
}
})
}

fn exists(
self: Arc<Self>,
options: ExistsOptions,
@@ -368,6 +439,32 @@ impl objects::ObjectImpl for Object {
}
}

fn replace_url_prefix<'a>(orig_url: &'a str, base: &str) -> Cow<'a, str> {
match url::Url::parse(orig_url) {
Ok(url) => {
let mut out = match url.path().is_empty() {
true => base.to_string(),
false => {
format!(
"{}/{}",
base.trim_end_matches('/'),
url.path().trim_start_matches("/")
)
}
};
if let Some(query) = url.query() {
out.push('?');
out.push_str(query);
}
Cow::Owned(out)
}
Err(_) => {
// If the input URL fails parsing, just don't do the replace
Cow::Borrowed(orig_url)
}
}
}

fn apply_upload_opts(opts: UploadOptions, req: &mut UploadObjectRequest, media: &mut Media) {
if let Some(content_type) = opts.content_type {
media.content_type = Cow::Owned(content_type);
26 changes: 26 additions & 0 deletions runtimes/core/src/objects/mod.rs
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ use futures::{Stream, StreamExt};
use std::borrow::Cow;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use std::{fmt::Debug, pin::Pin};
use thiserror::Error;
use tokio::io::AsyncRead;
@@ -51,6 +52,11 @@ trait ObjectImpl: Debug + Send + Sync {
options: UploadOptions,
) -> Pin<Box<dyn Future<Output = Result<ObjectAttrs, Error>> + Send>>;

fn signed_upload_url(
self: Arc<Self>,
options: UploadUrlOptions,
) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>>;

fn download(
self: Arc<Self>,
options: DownloadOptions,
@@ -223,6 +229,18 @@ impl Object {
}
}

pub async fn signed_upload_url(
&self,
options: UploadUrlOptions,
_source: Option<Arc<model::Request>>,
) -> Result<String, Error> {
const SEVEN_DAYS: Duration = Duration::new(7 * 86400, 0);
if options.ttl > SEVEN_DAYS {
return Err(Error::InvalidArgument);
}
self.imp.clone().signed_upload_url(options).await
}

pub fn download_stream(
&self,
options: DownloadOptions,
@@ -370,6 +388,9 @@ pub enum Error {
#[error("precondition failed")]
PreconditionFailed,

#[error("invalid argument")]
InvalidArgument,

#[error("internal error: {0:?}")]
Internal(anyhow::Error),

@@ -419,6 +440,11 @@ pub struct AttrsOptions {
pub version: Option<String>,
}

#[derive(Debug, Default)]
pub struct UploadUrlOptions {
pub ttl: Duration,
}

#[derive(Debug, Default)]
pub struct DeleteOptions {
pub version: Option<String>,
10 changes: 10 additions & 0 deletions runtimes/core/src/objects/noop/mod.rs
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ use crate::{encore::runtime::v1 as pb, EncoreName};

use super::{
AttrsOptions, DeleteOptions, DownloadOptions, ExistsOptions, ListOptions, PublicUrlError,
UploadUrlOptions,
};

#[derive(Debug)]
@@ -83,6 +84,15 @@ impl objects::ObjectImpl for Object {
))))
}

fn signed_upload_url(
self: Arc<Self>,
_options: UploadUrlOptions,
) -> Pin<Box<dyn Future<Output = Result<String, objects::Error>> + Send>> {
Box::pin(future::ready(Err(objects::Error::Internal(
anyhow::anyhow!("noop bucket does not support getting upload URL"),
))))
}

fn exists(
self: Arc<Self>,
_options: ExistsOptions,
27 changes: 26 additions & 1 deletion runtimes/core/src/objects/s3/bucket.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use async_stream::{stream, try_stream};
use aws_sdk_s3 as s3;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::presigning::PresigningConfig;
use aws_smithy_types::byte_stream::ByteStream;
use base64::Engine;
use bytes::{Bytes, BytesMut};
@@ -15,7 +16,7 @@ use tokio::io::{AsyncRead, AsyncReadExt};
use crate::encore::runtime::v1 as pb;
use crate::objects::{
self, AttrsOptions, DeleteOptions, DownloadOptions, Error, ExistsOptions, ListEntry,
ListOptions, ObjectAttrs, PublicUrlError,
ListOptions, ObjectAttrs, PublicUrlError, UploadUrlOptions,
};
use crate::{CloudName, EncoreName};

@@ -178,6 +179,30 @@ impl objects::ObjectImpl for Object {
})
}

fn signed_upload_url(
self: Arc<Self>,
options: UploadUrlOptions,
) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>> {
Box::pin(async move {
let client = self.bkt.client.get().await.clone();
let obj_name = self.bkt.obj_name(Cow::Borrowed(&self.name));

let res = client
.put_object()
.bucket(&self.bkt.cloud_name)
.key(obj_name)
.presigned(
PresigningConfig::expires_in(options.ttl)
.map_err(|e| Error::Other(e.into()))?,
)
.await;
match res {
Ok(req) => Ok(String::from(req.uri())),
Err(err) => Err(Error::Other(err.into())),
}
})
}

fn exists(
self: Arc<Self>,
options: ExistsOptions,
Loading

0 comments on commit 0457245

Please sign in to comment.