Add support for signed download urls. (#1715)

cli/daemon/objects/public.go

@@ -85,6 +85,14 @@ func (s *PublicBucketServer) handler(w http.ResponseWriter, req *http.Request) {
 	}
 	switch req.Method {
 	case "GET":
+		_, isSigned := (queryLowerCase(req))["x-goog-signature"]
+		if isSigned {
+			err := validateGcsSignedRequest(req, time.Now())
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+		}
 		obj, contents, err := store.Get("", bucketName, objName)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -103,9 +111,7 @@ func (s *PublicBucketServer) handler(w http.ResponseWriter, req *http.Request) {
 		w.Header().Set("Content-Length", strconv.Itoa(len(contents)))
 		w.Write(contents)
 	case "PUT":
-		// Only signed URLs are supported for PUT, and only GCS is supported
-		// for local development
-		err := validateGcsSignedUpload(req, time.Now())
+		err := validateGcsSignedRequest(req, time.Now())
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
@@ -136,14 +142,12 @@ func (s *PublicBucketServer) handler(w http.ResponseWriter, req *http.Request) {
 	}
 }
 
-func validateGcsSignedUpload(req *http.Request, now time.Time) error {
+// Only GCS is supported for local development
+func validateGcsSignedRequest(req *http.Request, now time.Time) error {
 	const dateLayout = "20060102T150405Z"
 	const gracePeriod = time.Duration(30) * time.Second
 
-	query := map[string]string{}
-	for k, vs := range req.URL.Query() {
-		query[strings.ToLower(k)] = vs[0]
-	}
+	query := queryLowerCase(req)
 
 	// We don't try to actually verify the signature, we only check that it's non-empty.
 
@@ -178,6 +182,14 @@ func validateGcsSignedUpload(req *http.Request, now time.Time) error {
 	return nil
 }
 
+func queryLowerCase(req *http.Request) map[string]string {
+	query := map[string]string{}
+	for k, vs := range req.URL.Query() {
+		query[strings.ToLower(k)] = vs[0]
+	}
+	return query
+}
+
 func parseObjectMeta(req *http.Request) storage.Object {
 	return storage.Object{ContentType: req.Header.Get("Content-Type")}
 }

proto/encore/parser/meta/v1/meta.proto

@@ -96,6 +96,9 @@ message BucketUsage {
     // Generating a signed URL to allow an external recipient to create or
     // update an object.
     SIGNED_UPLOAD_URL = 8;
+
+    // Generating a signed URL to allow an external recipient to download an object.
+    SIGNED_DOWNLOAD_URL = 9;
   }
 }
 

runtimes/core/src/objects/gcs/bucket.rs

@@ -14,8 +14,9 @@ use tokio::io::AsyncRead;
 
 use crate::encore::runtime::v1 as pb;
 use crate::objects::{
-    AttrsOptions, DeleteOptions, DownloadOptions, DownloadStream, Error, ExistsOptions, ListEntry,
-    ListOptions, ObjectAttrs, PublicUrlError, UploadOptions, UploadUrlOptions,
+    AttrsOptions, DeleteOptions, DownloadOptions, DownloadStream, DownloadUrlOptions, Error,
+    ExistsOptions, ListEntry, ListOptions, ObjectAttrs, PublicUrlError, UploadOptions,
+    UploadUrlOptions,
 };
 use crate::{objects, CloudName, EncoreName};
 use google_cloud_storage as gcs;
@@ -232,50 +233,26 @@ impl objects::ObjectImpl for Object {
         self: Arc<Self>,
         options: UploadUrlOptions,
     ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>> {
-        Box::pin(async move {
-            match self.bkt.client.get().await {
-                Ok(client) => {
-                    let gcs_opts = SignedURLOptions {
-                        method: gcs::sign::SignedURLMethod::PUT,
-                        expires: options.ttl,
-                        start_time: Some(SystemTime::now()),
-                        ..Default::default()
-                    };
-
-                    // We use a fake GCS service for local development. Ideally, the runtime
-                    // code would be oblivious to this once the GCS client is set up. But that
-                    // turns out to be difficult for URL signing, so we add a special case
-                    // here.
-                    let local_sign = &self.bkt.local_sign;
-                    let (access_id, sign_by) = match local_sign {
-                        Some(opt) => (
-                            Some(opt.access_id.clone()),
-                            Some(SignBy::PrivateKey(opt.private_key.as_bytes().to_vec())),
-                        ),
-                        None => (None, None),
-                    };
-
-                    let name = self.bkt.obj_name(Cow::Borrowed(&self.key)).into_owned();
-                    let mut url = client
-                        .signed_url(&self.bkt.cloud_name, &name, access_id, sign_by, gcs_opts)
-                        .await
-                        .map_err(|e| Error::Internal(e.into()))?;
-
-                    // More special handling for the local dev case.
-                    if let Some(cfg) = local_sign {
-                        url = replace_url_prefix(&url, &cfg.base_url)
-                            .into_owned()
-                            .to_string();
-                    }
+        let gcs_opts = SignedURLOptions {
+            method: gcs::sign::SignedURLMethod::PUT,
+            expires: options.ttl,
+            start_time: Some(SystemTime::now()),
+            ..Default::default()
+        };
+        self.signed_url(gcs_opts)
+    }
 
-                    Ok(url)
-                }
-                Err(err) => Err(Error::Internal(anyhow::anyhow!(
-                    "unable to resolve client: {}",
-                    err
-                ))),
-            }
-        })
+    fn signed_download_url(
+        self: Arc<Self>,
+        options: DownloadUrlOptions,
+    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>> {
+        let gcs_opts = SignedURLOptions {
+            method: gcs::sign::SignedURLMethod::GET,
+            expires: options.ttl,
+            start_time: Some(SystemTime::now()),
+            ..Default::default()
+        };
+        self.signed_url(gcs_opts)
     }
 
     fn exists(
@@ -439,6 +416,51 @@ impl objects::ObjectImpl for Object {
     }
 }
 
+impl Object {
+    fn signed_url(
+        self: Arc<Self>,
+        gcs_opts: SignedURLOptions,
+    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>> {
+        Box::pin(async move {
+            match self.bkt.client.get().await {
+                Ok(client) => {
+                    // We use a fake GCS service for local development. Ideally, the runtime
+                    // code would be oblivious to this once the GCS client is set up. But that
+                    // turns out to be difficult for URL signing, so we add a special case
+                    // here.
+                    let local_sign = &self.bkt.local_sign;
+                    let (access_id, sign_by) = match local_sign {
+                        Some(opt) => (
+                            Some(opt.access_id.clone()),
+                            Some(SignBy::PrivateKey(opt.private_key.as_bytes().to_vec())),
+                        ),
+                        None => (None, None),
+                    };
+
+                    let name = self.bkt.obj_name(Cow::Borrowed(&self.key)).into_owned();
+                    let mut url = client
+                        .signed_url(&self.bkt.cloud_name, &name, access_id, sign_by, gcs_opts)
+                        .await
+                        .map_err(|e| Error::Internal(e.into()))?;
+
+                    // More special handling for the local dev case.
+                    if let Some(cfg) = local_sign {
+                        url = replace_url_prefix(&url, &cfg.base_url)
+                            .into_owned()
+                            .to_string();
+                    }
+
+                    Ok(url)
+                }
+                Err(err) => Err(Error::Internal(anyhow::anyhow!(
+                    "unable to resolve client: {}",
+                    err
+                ))),
+            }
+        })
+    }
+}
+
 fn replace_url_prefix<'a>(orig_url: &'a str, base: &str) -> Cow<'a, str> {
     match url::Url::parse(orig_url) {
         Ok(url) => {

runtimes/core/src/objects/mod.rs

@@ -57,6 +57,11 @@ trait ObjectImpl: Debug + Send + Sync {
         options: UploadUrlOptions,
     ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>>;
 
+    fn signed_download_url(
+        self: Arc<Self>,
+        options: DownloadUrlOptions,
+    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>>;
+
     fn download(
         self: Arc<Self>,
         options: DownloadOptions,
@@ -241,6 +246,18 @@ impl Object {
         self.imp.clone().signed_upload_url(options).await
     }
 
+    pub async fn signed_download_url(
+        &self,
+        options: DownloadUrlOptions,
+        _source: Option<Arc<model::Request>>,
+    ) -> Result<String, Error> {
+        const SEVEN_DAYS: Duration = Duration::new(7 * 86400, 0);
+        if options.ttl > SEVEN_DAYS {
+            return Err(Error::InvalidArgument);
+        }
+        self.imp.clone().signed_download_url(options).await
+    }
+
     pub fn download_stream(
         &self,
         options: DownloadOptions,
@@ -445,6 +462,11 @@ pub struct UploadUrlOptions {
     pub ttl: Duration,
 }
 
+#[derive(Debug, Default)]
+pub struct DownloadUrlOptions {
+    pub ttl: Duration,
+}
+
 #[derive(Debug, Default)]
 pub struct DeleteOptions {
     pub version: Option<String>,

runtimes/core/src/objects/noop/mod.rs

@@ -9,8 +9,8 @@ use crate::objects;
 use crate::{encore::runtime::v1 as pb, EncoreName};
 
 use super::{
-    AttrsOptions, DeleteOptions, DownloadOptions, ExistsOptions, ListOptions, PublicUrlError,
-    UploadUrlOptions,
+    AttrsOptions, DeleteOptions, DownloadOptions, DownloadUrlOptions, ExistsOptions, ListOptions,
+    PublicUrlError, UploadUrlOptions,
 };
 
 #[derive(Debug)]
@@ -93,6 +93,15 @@ impl objects::ObjectImpl for Object {
         ))))
     }
 
+    fn signed_download_url(
+        self: Arc<Self>,
+        _options: DownloadUrlOptions,
+    ) -> Pin<Box<dyn Future<Output = Result<String, objects::Error>> + Send>> {
+        Box::pin(future::ready(Err(objects::Error::Internal(
+            anyhow::anyhow!("noop bucket does not support getting download URL"),
+        ))))
+    }
+
     fn exists(
         self: Arc<Self>,
         _options: ExistsOptions,

runtimes/core/src/objects/s3/bucket.rs

@@ -15,8 +15,8 @@ use tokio::io::{AsyncRead, AsyncReadExt};
 
 use crate::encore::runtime::v1 as pb;
 use crate::objects::{
-    self, AttrsOptions, DeleteOptions, DownloadOptions, Error, ExistsOptions, ListEntry,
-    ListOptions, ObjectAttrs, PublicUrlError, UploadUrlOptions,
+    self, AttrsOptions, DeleteOptions, DownloadOptions, DownloadUrlOptions, Error, ExistsOptions,
+    ListEntry, ListOptions, ObjectAttrs, PublicUrlError, UploadUrlOptions,
 };
 use crate::{CloudName, EncoreName};
 
@@ -203,6 +203,30 @@ impl objects::ObjectImpl for Object {
         })
     }
 
+    fn signed_download_url(
+        self: Arc<Self>,
+        options: DownloadUrlOptions,
+    ) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send>> {
+        Box::pin(async move {
+            let client = self.bkt.client.get().await.clone();
+            let obj_name = self.bkt.obj_name(Cow::Borrowed(&self.name));
+
+            let res = client
+                .get_object()
+                .bucket(&self.bkt.cloud_name)
+                .key(obj_name)
+                .presigned(
+                    PresigningConfig::expires_in(options.ttl)
+                        .map_err(|e| Error::Other(e.into()))?,
+                )
+                .await;
+            match res {
+                Ok(req) => Ok(String::from(req.uri())),
+                Err(err) => Err(Error::Other(err.into())),
+            }
+        })
+    }
+
     fn exists(
         self: Arc<Self>,
         options: ExistsOptions,

runtimes/go/storage/objects/bucket.go

@@ -406,6 +406,11 @@ type SignedUploadURL struct {
 	URL string
 }
 
+type SignedDownloadURL struct {
+	// The signed URL
+	URL string
+}
+
 // List lists objects in the bucket.
 func (b *Bucket) List(ctx context.Context, query *Query, options ...ListOption) iter.Seq2[*ListEntry, error] {
 	return func(yield func(*ListEntry, error) bool) {
@@ -608,6 +613,32 @@ func (b *Bucket) SignedUploadURL(ctx context.Context, object string, options ...
 	return &SignedUploadURL{URL: url}, nil
 }
 
+// Generates an external URL to allow uploading an object to the bucket.
+//
+// Anyone with possession of the URL can write to the given object name
+// without any additional auth.
+func (b *Bucket) SignedDownloadURL(ctx context.Context, object string, options ...DownloadURLOption) (*SignedDownloadURL, error) {
+	var opt downloadURLOptions
+	for _, o := range options {
+		o.applyDownloadURL(&opt)
+	}
+	if opt.TTL == 0 {
+		opt.TTL = time.Hour
+	}
+	if opt.TTL > 7*24*time.Hour {
+		return nil, types.ErrInvalidArgument
+	}
+	url, err := b.impl.SignedDownloadURL(types.DownloadURLData{
+		Ctx:    ctx,
+		Object: b.toCloudObject(object),
+		TTL:    opt.TTL,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &SignedDownloadURL{URL: url}, nil
+}
+
 // Exists reports whether an object exists in the bucket.
 func (b *Bucket) Exists(ctx context.Context, object string, options ...ExistsOption) (bool, error) {
 	var opt existsOptions

runtimes/go/storage/objects/internal/providers/gcs/bucket.go

@@ -196,7 +196,19 @@ func (b *bucket) SignedUploadURL(data types.UploadURLData) (string, error) {
 		Method:  "PUT",
 		Expires: time.Now().Add(data.TTL),
 	}
+	return b.signedURL(data.Object.String(), opts)
+}
+
+func (b *bucket) SignedDownloadURL(data types.DownloadURLData) (string, error) {
+	opts := &storage.SignedURLOptions{
+		Scheme:  storage.SigningSchemeV4,
+		Method:  "GET",
+		Expires: time.Now().Add(data.TTL),
+	}
+	return b.signedURL(data.Object.String(), opts)
+}
 
+func (b *bucket) signedURL(object string, opts *storage.SignedURLOptions) (string, error) {
 	// We use a fake GCS service for local development. Ideally, the runtime
 	// code would be oblivious to this once the GCS client is set up. But that
 	// turns out to be difficult for URL signing, so we add a special case
@@ -206,7 +218,7 @@ func (b *bucket) SignedUploadURL(data types.UploadURLData) (string, error) {
 		opts.PrivateKey = []byte(b.localSign.privateKey)
 	}
 
-	url, err := b.handle.SignedURL(data.Object.String(), opts)
+	url, err := b.handle.SignedURL(object, opts)
 	if err != nil {
 		return "", mapErr(err)
 	}

runtimes/go/storage/objects/internal/providers/noop/noop.go

@@ -36,3 +36,7 @@ func (b *BucketImpl) Attrs(data types.AttrsData) (*types.ObjectAttrs, error) {
 func (b *BucketImpl) SignedUploadURL(data types.UploadURLData) (string, error) {
 	return "", fmt.Errorf("cannot get upload url from noop bucket")
 }
+
+func (b *BucketImpl) SignedDownloadURL(data types.DownloadURLData) (string, error) {
+	return "", fmt.Errorf("cannot get download url from noop bucket")
+}