add migration to convert array to jsonb

packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs

@@ -1,7 +1,11 @@
-use model::entity::{antenna, newtype::StringVec};
+use model::entity::{
+    access_token, antenna, app, emoji, gallery_post, hashtag, messaging_message, meta,
+    newtype::{I32Vec, StringVec},
+    note, note_edit, poll, registry_item, user, user_profile, webhook,
+};
 use sea_orm_migration::{
     prelude::*,
-    sea_orm::{DbBackend, EntityTrait, Statement},
+    sea_orm::{DbBackend, EntityTrait, Statement, TryGetable},
 };
 use serde_json::json;
 
@@ -11,50 +15,220 @@ pub struct Migration;
 #[async_trait::async_trait]
 impl MigrationTrait for Migration {
     async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .drop_table(Table::drop().table(Alias::new("reversi_game")).to_owned())
+            .await?;
+
+        manager
+            .drop_table(
+                Table::drop()
+                    .table(Alias::new("reversi_matching"))
+                    .to_owned(),
+            )
+            .await?;
+
         if manager.get_database_backend() == DbBackend::Sqlite {
             return Ok(());
         }
 
         let db = manager.get_connection();
-        let query = Query::select()
-            .column(Antenna::Id)
-            .column(Antenna::Users)
-            .from(Antenna::Table)
-            .to_string(PostgresQueryBuilder);
-        let res: Vec<(String, Vec<String>)> = db
-            .query_all(Statement::from_string(DbBackend::Postgres, query))
-            .await?
-            .iter()
-            .filter_map(|r| r.try_get_many_by_index().ok())
-            .collect();
 
-        manager
+        macro_rules! copy_data {
-            .alter_table(
+            ($data:ident, $ent:ident, $col:tt) => {
-                Table::alter()
+                let models: Vec<$ent::ActiveModel> = $data
-                    .table(Antenna::Table)
+                    .iter()
-                    .drop_column(Antenna::Users)
+                    .map(|(id, r)| $ent::ActiveModel {
-                    .add_column(
+                        id: sea_orm::Set(id.to_owned()),
-                        ColumnDef::new(Antenna::Users)
+                        $col: sea_orm::Set(StringVec::from(r.to_owned())),
-                            .json_binary()
+                        ..Default::default()
-                            .not_null()
+                    })
-                            .default(json!([])),
+                    .collect();
-                    )
+                for model in models {
-                    .to_owned(),
+                    $ent::Entity::update(model).exec(db).await?;
-            )
+                }
-            .await?;
+            };
+        }
 
-        let models: Vec<antenna::ActiveModel> = res
+        macro_rules! convert_to_stringvec_json {
+            ($table:expr, $id:expr, $col:expr, $ent:ident, $col_name:tt) => {
+                let query = select_query($table, $id, $col);
+                let res = get_vec::<Vec<String>>(db, query).await?;
+                convert_col(manager, $table, $col).await?;
+                copy_data!(res, $ent, $col_name);
+            };
+        }
+
+        convert_to_stringvec_json!(
+            AccessToken::Table,
+            AccessToken::Id,
+            AccessToken::Permission,
+            access_token,
+            permission
+        );
+        convert_to_stringvec_json!(Antenna::Table, Antenna::Id, Antenna::Users, antenna, users);
+        convert_to_stringvec_json!(App::Table, App::Id, App::Permission, app, permission);
+        convert_to_stringvec_json!(Emoji::Table, Emoji::Id, Emoji::Aliases, emoji, aliases);
+        convert_to_stringvec_json!(
+            GalleryPost::Table,
+            GalleryPost::Id,
+            GalleryPost::FileIds,
+            gallery_post,
+            file_ids
+        );
+        convert_to_stringvec_json!(
+            GalleryPost::Table,
+            GalleryPost::Id,
+            GalleryPost::Tags,
+            gallery_post,
+            tags
+        );
+        convert_to_stringvec_json!(
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::MentionedUserIds,
+            hashtag,
+            mentioned_user_ids
+        );
+        convert_to_stringvec_json!(
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::MentionedLocalUserIds,
+            hashtag,
+            mentioned_local_user_ids
+        );
+        convert_to_stringvec_json!(
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::MentionedRemoteUserIds,
+            hashtag,
+            mentioned_remote_user_ids
+        );
+        convert_to_stringvec_json!(
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::AttachedUserIds,
+            hashtag,
+            attached_user_ids
+        );
+        convert_to_stringvec_json!(
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::AttachedLocalUserIds,
+            hashtag,
+            attached_local_user_ids
+        );
+        convert_to_stringvec_json!(
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::AttachedRemoteUserIds,
+            hashtag,
+            attached_remote_user_ids
+        );
+        convert_to_stringvec_json!(
+            MessagingMessage::Table,
+            MessagingMessage::Id,
+            MessagingMessage::Reads,
+            messaging_message,
+            reads
+        );
+        convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::Langs, meta, langs);
+        convert_to_stringvec_json!(
+            Meta::Table,
+            Meta::Id,
+            Meta::BlockedHosts,
+            meta,
+            blocked_hosts
+        );
+        convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::HiddenTags, meta, hidden_tags);
+        convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::PinnedUsers, meta, pinned_users);
+        convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::PinnedPages, meta, pinned_pages);
+        convert_to_stringvec_json!(
+            Meta::Table,
+            Meta::Id,
+            Meta::RecommendedInstances,
+            meta,
+            recommended_instances
+        );
+        convert_to_stringvec_json!(
+            Meta::Table,
+            Meta::Id,
+            Meta::SilencedHosts,
+            meta,
+            silenced_hosts
+        );
+        convert_to_stringvec_json!(Note::Table, Note::Id, Note::FileIds, note, file_ids);
+        convert_to_stringvec_json!(
+            Note::Table,
+            Note::Id,
+            Note::AttachedFileTypes,
+            note,
+            attached_file_types
+        );
+        convert_to_stringvec_json!(
+            Note::Table,
+            Note::Id,
+            Note::VisibleUserIds,
+            note,
+            visible_user_ids
+        );
+        convert_to_stringvec_json!(Note::Table, Note::Id, Note::Mentions, note, mentions);
+        convert_to_stringvec_json!(Note::Table, Note::Id, Note::Emojis, note, emojis);
+        convert_to_stringvec_json!(Note::Table, Note::Id, Note::Tags, note, tags);
+        convert_to_stringvec_json!(
+            NoteEdit::Table,
+            NoteEdit::Id,
+            NoteEdit::FileIds,
+            note_edit,
+            file_ids
+        );
+
+        // Convert poll here because its primary key is not id, but note_id.
+        let query = select_query(Poll::Table, Poll::NoteId, Poll::Choices);
+        let res = get_vec::<Vec<String>>(db, query).await?;
+        convert_col(manager, Poll::Table, Poll::Choices).await?;
+        let poll_models: Vec<poll::ActiveModel> = res
             .iter()
-            .map(|(id, users)| antenna::ActiveModel {
+            .map(|(id, r)| poll::ActiveModel {
-                id: sea_orm::Set(id.to_owned()),
+                note_id: sea_orm::Set(id.to_owned()),
-                users: sea_orm::Set(StringVec::from(users.to_owned())),
+                choices: sea_orm::Set(StringVec::from(r.to_owned())),
                 ..Default::default()
             })
             .collect();
-
+        for model in poll_models {
-        for model in models {
+            poll::Entity::update(model).exec(db).await?;
-            antenna::Entity::update(model).exec(db).await?;
         }
+        let query = select_query(Poll::Table, Poll::NoteId, Poll::Votes);
+        let res = get_vec::<Vec<i32>>(db, query).await?;
+        convert_col(manager, Poll::Table, Poll::Votes).await?;
+        let poll_models: Vec<poll::ActiveModel> = res
+            .iter()
+            .map(|(id, r)| poll::ActiveModel {
+                note_id: sea_orm::Set(id.to_owned()),
+                votes: sea_orm::Set(I32Vec::from(r.to_owned())),
+                ..Default::default()
+            })
+            .collect();
+        for model in poll_models {
+            poll::Entity::update(model).exec(db).await?;
+        }
+
+        convert_to_stringvec_json!(
+            RegistryItem::Table,
+            RegistryItem::Id,
+            RegistryItem::Scope,
+            registry_item,
+            scope
+        );
+        convert_to_stringvec_json!(User::Table, User::Id, User::Tags, user, tags);
+        convert_to_stringvec_json!(User::Table, User::Id, User::Emojis, user, emojis);
+        convert_to_stringvec_json!(
+            UserProfile::Table,
+            UserProfile::Id,
+            UserProfile::MutingNotificationTypes,
+            user_profile,
+            muting_notification_types
+        );
+        convert_to_stringvec_json!(Webhook::Table, Webhook::Id, Webhook::On, webhook, on);
 
         Ok(())
     }
@@ -66,9 +240,160 @@ impl MigrationTrait for Migration {
 }
 
 /// Learn more at https://docs.rs/sea-query#iden
-#[derive(Iden)]
+#[derive(Iden, Clone)]
 enum Antenna {
     Table,
     Id,
     Users,
 }
+#[derive(Iden, Clone)]
+enum AccessToken {
+    Table,
+    Id,
+    Permission,
+}
+#[derive(Iden, Clone)]
+enum App {
+    Table,
+    Id,
+    Permission,
+}
+#[derive(Iden, Clone)]
+enum Emoji {
+    Table,
+    Id,
+    Aliases,
+}
+#[derive(Iden, Clone)]
+enum GalleryPost {
+    Table,
+    Id,
+    FileIds,
+    Tags,
+}
+#[derive(Iden, Clone)]
+enum Hashtag {
+    Table,
+    Id,
+    MentionedUserIds,
+    MentionedLocalUserIds,
+    MentionedRemoteUserIds,
+    AttachedUserIds,
+    AttachedLocalUserIds,
+    AttachedRemoteUserIds,
+}
+#[derive(Iden, Clone)]
+enum MessagingMessage {
+    Table,
+    Id,
+    Reads,
+}
+#[derive(Iden, Clone)]
+enum Meta {
+    Table,
+    Id,
+    Langs,
+    HiddenTags,
+    BlockedHosts,
+    PinnedUsers,
+    PinnedPages,
+    RecommendedInstances,
+    SilencedHosts,
+}
+#[derive(Iden, Clone)]
+enum Note {
+    Table,
+    Id,
+    FileIds,
+    AttachedFileTypes,
+    VisibleUserIds,
+    Mentions,
+    Emojis,
+    Tags,
+}
+#[derive(Iden, Clone)]
+enum NoteEdit {
+    Table,
+    Id,
+    FileIds,
+}
+#[derive(Iden, Clone)]
+enum Page {
+    Table,
+    Id,
+    VisibleUserIds,
+}
+#[derive(Iden, Clone)]
+enum Poll {
+    Table,
+    NoteId,
+    Choices,
+    Votes, // I32Vec
+}
+#[derive(Iden, Clone)]
+enum RegistryItem {
+    Table,
+    Id,
+    Scope,
+}
+#[derive(Iden, Clone)]
+enum User {
+    Table,
+    Id,
+    Tags,
+    Emojis,
+}
+#[derive(Iden, Clone)]
+enum UserProfile {
+    Table,
+    Id,
+    MutingNotificationTypes,
+}
+#[derive(Iden, Clone)]
+enum Webhook {
+    Table,
+    Id,
+    On,
+}
+
+fn select_query<T: Iden + 'static>(table: T, id: T, col: T) -> String {
+    Query::select()
+        .column(id)
+        .column(col)
+        .from(table)
+        .to_string(PostgresQueryBuilder)
+}
+
+async fn get_vec<'a, T: TryGetable>(
+    db: &SchemaManagerConnection<'a>,
+    query: String,
+) -> Result<Vec<(String, T)>, DbErr> {
+    let res: Vec<(String, T)> = db
+        .query_all(Statement::from_string(DbBackend::Postgres, query))
+        .await?
+        .iter()
+        .filter_map(|r| r.try_get_many_by_index().ok())
+        .collect();
+    Ok(res)
+}
+
+async fn convert_col<'a, T: Iden + Clone + 'static>(
+    manager: &SchemaManager<'a>,
+    table: T,
+    col: T,
+) -> Result<(), DbErr> {
+    manager
+        .alter_table(
+            Table::alter()
+                .table(table)
+                .drop_column(col.to_owned())
+                .add_column(
+                    ColumnDef::new(col.to_owned())
+                        .json_binary()
+                        .not_null()
+                        .default(json!([])),
+                )
+                .to_owned(),
+        )
+        .await
+}

packages/backend/native-utils/crates/model/Cargo.toml

@@ -5,8 +5,8 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [features]
-default = ["legacy"]
+default = []
-legacy = ["sea-orm/postgres-array"]
+legacy = []
 
 [dependencies]
 async-trait = "0.1.68"
@@ -18,7 +18,7 @@ jsonschema = "0.17.0"
 once_cell = "1.17.1"
 parse-display = "0.8.0"
 schemars = { version = "0.8.12", features = ["chrono"] }
-sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls"] }
+sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "postgres-array", "sqlx-sqlite", "runtime-tokio-rustls"] }
 serde = { version = "1.0.163", features = ["derive"] }
 serde_json = "1.0.96"
 thiserror = "1.0.40"

packages/backend/native-utils/crates/model/src/entity/mod.rs

@@ -53,8 +53,6 @@ pub mod registration_ticket;
 pub mod registry_item;
 pub mod relay;
 pub mod renote_muting;
-pub mod reversi_game;
-pub mod reversi_matching;
 pub mod sea_orm_active_enums;
 pub mod signin;
 pub mod sw_subscription;

packages/backend/native-utils/crates/model/src/entity/prelude.rs

@@ -50,8 +50,6 @@ pub use super::registration_ticket::Entity as RegistrationTicket;
 pub use super::registry_item::Entity as RegistryItem;
 pub use super::relay::Entity as Relay;
 pub use super::renote_muting::Entity as RenoteMuting;
-pub use super::reversi_game::Entity as ReversiGame;
-pub use super::reversi_matching::Entity as ReversiMatching;
 pub use super::signin::Entity as Signin;
 pub use super::sw_subscription::Entity as SwSubscription;
 pub use super::used_username::Entity as UsedUsername;

packages/backend/native-utils/crates/model/src/entity/reversi_game.rs

@@ -1,69 +0,0 @@
-//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
-
-use sea_orm::entity::prelude::*;
-
-use super::newtype::StringVec;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "reversi_game")]
-pub struct Model {
-    #[sea_orm(primary_key, auto_increment = false)]
-    pub id: String,
-    #[sea_orm(column_name = "createdAt")]
-    pub created_at: DateTimeWithTimeZone,
-    #[sea_orm(column_name = "startedAt")]
-    pub started_at: Option<DateTimeWithTimeZone>,
-    #[sea_orm(column_name = "user1Id")]
-    pub user1_id: String,
-    #[sea_orm(column_name = "user2Id")]
-    pub user2_id: String,
-    #[sea_orm(column_name = "user1Accepted")]
-    pub user1_accepted: bool,
-    #[sea_orm(column_name = "user2Accepted")]
-    pub user2_accepted: bool,
-    pub black: Option<i32>,
-    #[sea_orm(column_name = "isStarted")]
-    pub is_started: bool,
-    #[sea_orm(column_name = "isEnded")]
-    pub is_ended: bool,
-    #[sea_orm(column_name = "winnerId")]
-    pub winner_id: Option<String>,
-    pub surrendered: Option<String>,
-    #[sea_orm(column_type = "JsonBinary")]
-    pub logs: Json,
-    pub map: StringVec,
-    pub bw: String,
-    #[sea_orm(column_name = "isLlotheo")]
-    pub is_llotheo: bool,
-    #[sea_orm(column_name = "canPutEverywhere")]
-    pub can_put_everywhere: bool,
-    #[sea_orm(column_name = "loopedBoard")]
-    pub looped_board: bool,
-    #[sea_orm(column_type = "JsonBinary", nullable)]
-    pub form1: Option<Json>,
-    #[sea_orm(column_type = "JsonBinary", nullable)]
-    pub form2: Option<Json>,
-    pub crc32: Option<String>,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-    #[sea_orm(
-        belongs_to = "super::user::Entity",
-        from = "Column::User2Id",
-        to = "super::user::Column::Id",
-        on_update = "NoAction",
-        on_delete = "Cascade"
-    )]
-    User2,
-    #[sea_orm(
-        belongs_to = "super::user::Entity",
-        from = "Column::User1Id",
-        to = "super::user::Column::Id",
-        on_update = "NoAction",
-        on_delete = "Cascade"
-    )]
-    User1,
-}
-
-impl ActiveModelBehavior for ActiveModel {}

packages/backend/native-utils/crates/model/src/entity/reversi_matching.rs

@@ -1,38 +0,0 @@
-//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
-
-use sea_orm::entity::prelude::*;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "reversi_matching")]
-pub struct Model {
-    #[sea_orm(primary_key, auto_increment = false)]
-    pub id: String,
-    #[sea_orm(column_name = "createdAt")]
-    pub created_at: DateTimeWithTimeZone,
-    #[sea_orm(column_name = "parentId")]
-    pub parent_id: String,
-    #[sea_orm(column_name = "childId")]
-    pub child_id: String,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-    #[sea_orm(
-        belongs_to = "super::user::Entity",
-        from = "Column::ParentId",
-        to = "super::user::Column::Id",
-        on_update = "NoAction",
-        on_delete = "Cascade"
-    )]
-    User2,
-    #[sea_orm(
-        belongs_to = "super::user::Entity",
-        from = "Column::ChildId",
-        to = "super::user::Column::Id",
-        on_update = "NoAction",
-        on_delete = "Cascade"
-    )]
-    User1,
-}
-
-impl ActiveModelBehavior for ActiveModel {}

packages/backend/native-utils/crates/model/tests/common.rs

@@ -31,7 +31,7 @@ async fn setup_schema(db: DbConn) {
     let stmt = schema.create_table_from_entity(antenna::Entity);
     db.execute(db.get_database_backend().build(&stmt))
         .await
-        .expect("Unable to initialize in-memoty sqlite");
+        .expect("Unable to setup schemas for in-memoty sqlite");
 }
 
 /// Delete all entries in the database.
@@ -101,9 +101,7 @@ async fn setup_model(db: &DbConn) {
 mod int_test {
     use sea_orm::Database;
 
-    use crate::setup_schema;
+    use super::{cleanup, prepare, setup_schema};
-
-    use super::{cleanup, prepare};
 
     #[tokio::test]
     async fn can_prepare_and_cleanup() {