[WJ-1196] Add support for S3 presign URLs to upload blobs

deepwell/config.example.toml

@@ -355,6 +355,22 @@ minimum-name-bytes = 3
 # Set to 0 to disable.
 refill-name-change-days = 90
 
+
+[file]
+
+# The length of paths used for S3 presigned URLs.
+#
+# The value doesn't particularly matter so long as it is sufficiently long
+# to avoid collisions.
+#
+# Just to be safe, the generation mechanism is the same as for session tokens.
+presigned-path-length = 32
+
+# How long a presigned URL lasts before expiry.
+#
+# The value should only be a few minutes, and no longer than 12 hours.
+presigned-expiration-minutes = 5
+
 [message]
 
 # The maximum size of a message's subject line, in bytes.

deepwell/migrations/20220906103252_deepwell.sql

@@ -411,6 +411,18 @@ CREATE TABLE page_vote (
     CHECK ((disabled_at IS NULL) = (disabled_by IS NULL))
 );
 
+--
+-- Blobs
+--
+
+-- Manages blobs that are being uploaded by the user
+CREATE TABLE blob_pending (
+    pending_file_id BIGSERIAL PRIMARY KEY,
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+    s3_path TEXT NOT NULL CHECK (length(s3_path) > 1),
+    presign_url TEXT NOT NULL CHECK (length(presign_url) > 1)
+);
+
 --
 -- Files
 --
@@ -439,6 +451,7 @@ CREATE TABLE file (
     name TEXT NOT NULL,
     page_id BIGINT NOT NULL REFERENCES page(page_id),
     site_id BIGINT NOT NULL REFERENCES site(site_id),
+    pending_blob_id BIGINT REFERENCES file_pending(pending_file_id),
 
     UNIQUE (page_id, name, deleted_at)
 );
@@ -514,7 +527,7 @@ CREATE TYPE message_recipient_type AS ENUM (
 -- A "record" is the underlying message data, with its contents, attachments,
 -- and associated metadata such as sender and recipient(s).
 CREATE TABLE message_record (
-    external_id TEXT PRIMARY KEY,
+    external_id TEXT PRIMARY KEY, -- ID comes from message_draft
     created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
     drafted_at TIMESTAMP WITH TIME ZONE NOT NULL,
     retracted_at TIMESTAMP WITH TIME ZONE,

deepwell/src/api.rs

@@ -257,7 +257,10 @@ async fn build_module(app_state: ServerState) -> anyhow::Result<RpcModule<Server
     register!("blob_get", blob_get);
 
     // Files
-    register!("file_upload", file_upload);
+    register!("file_create_start", file_create_start);
+    register!("file_create_finish", file_create_finish);
+    register!("file_edit_start", file_edit_start);
+    register!("file_edit_finish", file_edit_finish);
     register!("file_get", file_get);
     register!("file_edit", file_edit);
     register!("file_delete", file_delete);

deepwell/src/config/file.rs

@@ -53,6 +53,7 @@ pub struct ConfigFile {
     ftml: Ftml,
     special_pages: SpecialPages,
     user: User,
+    file: FileSection,
     message: Message,
 }
 
@@ -181,6 +182,14 @@ struct User {
     minimum_name_bytes: usize,
 }
 
+// NOTE: Name conflict with std::fs::File
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
+struct FileSection {
+    presigned_path_length: usize,
+    presigned_expiration_minutes: u32,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "kebab-case")]
 struct Message {
@@ -303,6 +312,11 @@ impl ConfigFile {
                     refill_name_change_days,
                     minimum_name_bytes,
                 },
+            file:
+                FileSection {
+                    presigned_path_length,
+                    presigned_expiration_minutes,
+                },
             message:
                 Message {
                     maximum_subject_bytes: maximum_message_subject_bytes,
@@ -424,6 +438,8 @@ impl ConfigFile {
                 ))
             },
             minimum_name_bytes,
+            presigned_path_length,
+            presigned_expiry_secs: presigned_expiration_minutes * 60,
             maximum_message_subject_bytes,
             maximum_message_body_bytes,
             maximum_message_recipients,

deepwell/src/config/object.rs

@@ -200,6 +200,12 @@ pub struct Config {
     /// Minimum length of bytes in a username.
     pub minimum_name_bytes: usize,
 
+    /// Length of randomly-generated portion of S3 presigned URLs.
+    pub presigned_path_length: usize,
+
+    /// How long S3 presigned URLs will last before expiry.
+    pub presigned_expiry_secs: u32,
+
     /// Maximum size of the subject line allowed in a direct message.
     pub maximum_message_subject_bytes: usize,
 

deepwell/src/endpoints/file.rs

@@ -21,11 +21,11 @@
 use super::prelude::*;
 use crate::models::file::Model as FileModel;
 use crate::models::file_revision::Model as FileRevisionModel;
-use crate::services::blob::BlobService;
+use crate::services::blob::{BlobMetadata, BlobService, GetBlobOutput};
 use crate::services::file::{
-    DeleteFile, DeleteFileOutput, EditFile, EditFileOutput, GetBlobOutput,
-    GetFileDetails, GetFileOutput, MoveFile, MoveFileOutput, RestoreFile,
-    RestoreFileOutput, UploadFile, UploadFileOutput,
+    DeleteFile, DeleteFileOutput, EditFile, EditFileOutput, FinishFileCreation,
+    FinishFileCreationOutput, GetFileDetails, GetFileOutput, MoveFile, MoveFileOutput,
+    RestoreFile, RestoreFileOutput, StartFileCreation, StartFileCreationOutput,
 };
 use crate::services::Result;
 use crate::web::{Bytes, FileDetails};
@@ -40,14 +40,19 @@ pub async fn blob_get(
     info!("Getting blob for S3 hash");
     let hash: Bytes = params.parse()?;
     let data = BlobService::get(ctx, hash.as_ref()).await?;
-    let metadata = BlobService::get_metadata(ctx, hash.as_ref()).await?;
 
-    let output = GetBlobOutput {
+    let BlobMetadata {
+        mime,
+        size,
+        created_at,
+    } = BlobService::get_metadata(ctx, hash.as_ref()).await?;
+
+    Ok(GetBlobOutput {
         data,
-        mime: metadata.mime,
-        size: metadata.size,
-    };
-    Ok(output)
+        mime,
+        size,
+        created_at,
+    })
 }
 
 pub async fn file_get(
@@ -79,21 +84,53 @@ pub async fn file_get(
     }
 }
 
-pub async fn file_upload(
+pub async fn file_create_start(
+    ctx: &ServiceContext<'_>,
+    params: Params<'static>,
+) -> Result<StartFileCreationOutput> {
+    let input: StartFileCreation = params.parse()?;
+
+    info!(
+        "Starting file upload '{}' to page ID {} in site ID {}",
+        input.name, input.page_id, input.site_id,
+    );
+
+    FileService::start_new_upload(ctx, input).await
+}
+
+pub async fn file_create_finish(
     ctx: &ServiceContext<'_>,
     params: Params<'static>,
-) -> Result<UploadFileOutput> {
-    let input: UploadFile = params.parse()?;
+) -> Result<FinishFileCreationOutput> {
+    let input: FinishFileCreation = params.parse()?;
 
     info!(
-        "Uploading file '{}' ({} bytes) to page ID {} in site ID {}",
-        input.name,
-        input.data.len(),
+        "Finishing file upload (pending blob ID {} for file ID {} in page ID {} in site ID {}",
+        input.pending_blob_id,
+        input.file_id,
         input.page_id,
         input.site_id,
     );
 
-    FileService::upload(ctx, input).await
+    FileService::finish_new_upload(ctx, input).await
+}
+
+// TODO
+pub async fn file_edit_start(
+    ctx: &ServiceContext<'_>,
+    _params: Params<'static>,
+) -> Result<()> {
+    let _ = FileService::start_edit_upload(ctx).await?;
+    todo!()
+}
+
+// TODO
+pub async fn file_edit_finish(
+    ctx: &ServiceContext<'_>,
+    _params: Params<'static>,
+) -> Result<()> {
+    let _ = FileService::finish_edit_upload(ctx).await?;
+    todo!()
 }
 
 pub async fn file_edit(

deepwell/src/models/blob_pending.rs

@@ -0,0 +1,21 @@
+use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
+#[sea_orm(table_name = "blob_pending")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub pending_blob_id: i64,
+    pub created_at: TimeDateTimeWithTimeZone,
+
+    #[sea_orm(column_type = "Text")]
+    pub s3_path: String,
+
+    #[sea_orm(column_type = "Text")]
+    pub presign_url: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}

deepwell/src/models/file.rs

@@ -16,6 +16,7 @@ pub struct Model {
     pub name: String,
     pub page_id: i64,
     pub site_id: i64,
+    pub pending_blob_id: Option<i64>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

deepwell/src/models/mod.rs

@@ -3,6 +3,7 @@
 pub mod prelude;
 
 pub mod alias;
+pub mod blob_pending;
 pub mod file;
 pub mod file_revision;
 pub mod filter;

deepwell/src/services/blob/mod.rs

@@ -27,6 +27,9 @@
 #[allow(unused_imports)]
 mod prelude {
     pub use super::super::prelude::*;
+    pub use super::service::{
+        EMPTY_BLOB_HASH, EMPTY_BLOB_MIME, EMPTY_BLOB_TIMESTAMP, PRESIGN_DIRECTORY,
+    };
     pub use super::structs::*;
     pub use crate::hash::{blob_hash_to_hex, sha512_hash, BlobHash};
 }
@@ -36,5 +39,8 @@ mod service;
 mod structs;
 
 pub use self::mime::MimeAnalyzer;
-pub use self::service::BlobService;
+pub use self::service::{
+    BlobService, EMPTY_BLOB_HASH, EMPTY_BLOB_MIME, EMPTY_BLOB_TIMESTAMP,
+    PRESIGN_DIRECTORY,
+};
 pub use self::structs::*;