From bb28463e6b75264dc97b2874a4b15d3561c23152 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 24 Sep 2024 10:58:37 +0200 Subject: [PATCH 1/7] chore: use _supabase database for internals This is to avoid overloading our user postgres database with every new addition to _analytics or _realtime --- internal/db/dump/dump.go | 3 --- internal/db/reset/reset.go | 2 ++ internal/db/start/start.go | 2 +- internal/db/start/templates/schema.sql | 24 ++++++++++++++++-------- internal/start/start.go | 8 ++++---- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/internal/db/dump/dump.go b/internal/db/dump/dump.go index a52b1dc41..e75cbb8b9 100644 --- a/internal/db/dump/dump.go +++ b/internal/db/dump/dump.go @@ -100,9 +100,6 @@ func dumpData(ctx context.Context, config pgconn.Config, schema, excludeTable [] // "storage", // "supabase_functions", "supabase_migrations", - "_analytics", - "_realtime", - "_supavisor", } var env []string if len(schema) > 0 { diff --git a/internal/db/reset/reset.go b/internal/db/reset/reset.go index 62c3d3350..89a33fda5 100644 --- a/internal/db/reset/reset.go +++ b/internal/db/reset/reset.go @@ -170,7 +170,9 @@ func recreateDatabase(ctx context.Context, options ...func(*pgx.ConnConfig)) err sql := migration.MigrationFile{ Statements: []string{ "DROP DATABASE IF EXISTS postgres WITH (FORCE)", + "DROP DATABASE IF EXISTS _supabase WITH (FORCE)", "CREATE DATABASE postgres WITH OWNER postgres", + "CREATE DATABASE _supabase WITH OWNER postgres", }, } return sql.ExecBatch(ctx, conn) diff --git a/internal/db/start/start.go b/internal/db/start/start.go index 3039b04d9..c25aa24e1 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -228,7 +228,7 @@ func initRealtimeJob(host string) utils.DockerJob { "DB_PORT=5432", "DB_USER=supabase_admin", "DB_PASSWORD=" + utils.Config.Db.Password, - "DB_NAME=postgres", + "DB_NAME=_supabase", "DB_AFTER_CONNECT_QUERY=SET search_path TO _realtime", "DB_ENC_KEY=" + utils.Config.Realtime.EncryptionKey, "API_JWT_SECRET=" + utils.Config.Auth.JwtSecret, diff --git a/internal/db/start/templates/schema.sql b/internal/db/start/templates/schema.sql index 0d0010231..418f0ee05 100644 --- a/internal/db/start/templates/schema.sql +++ b/internal/db/start/templates/schema.sql @@ -12,14 +12,22 @@ ALTER USER supabase_storage_admin WITH PASSWORD :'pgpass'; ALTER USER supabase_replication_admin WITH PASSWORD :'pgpass'; ALTER USER supabase_read_only_user WITH PASSWORD :'pgpass'; -create schema if not exists _realtime; -alter schema _realtime owner to postgres; - -create schema if not exists _analytics; -alter schema _analytics owner to postgres; - -create schema if not exists _supavisor; -alter schema _supavisor owner to postgres; +CREATE DATABASE _supabase WITH OWNER postgres; +-- Connect to the _supabase database +\c _supabase +-- Create schemas in _supabase database for +-- internals tools and reports to not overload user database +-- with non-user activity +CREATE SCHEMA IF NOT EXISTS _realtime; +ALTER SCHEMA _realtime OWNER TO postgres; + +CREATE SCHEMA IF NOT EXISTS _analytics; +ALTER SCHEMA _analytics OWNER TO postgres; + +CREATE SCHEMA IF NOT EXISTS _supavisor; +ALTER SCHEMA _supavisor OWNER TO postgres; +-- Switch back to the main database +\c postgres BEGIN; diff --git a/internal/start/start.go b/internal/start/start.go index 2c3017ce5..413b8e2ed 100644 --- a/internal/start/start.go +++ b/internal/start/start.go @@ -195,7 +195,7 @@ func run(p utils.Program, ctx context.Context, fsys afero.Fs, excludedContainers // Start Logflare if utils.Config.Analytics.Enabled && !isContainerExcluded(utils.Config.Analytics.Image, excluded) { env := []string{ - "DB_DATABASE=" + dbConfig.Database, + "DB_DATABASE=_supabase", "DB_HOSTNAME=" + dbConfig.Host, fmt.Sprintf("DB_PORT=%d", dbConfig.Port), "DB_SCHEMA=_analytics", @@ -228,7 +228,7 @@ func run(p utils.Program, ctx context.Context, fsys afero.Fs, excludedContainers ) case config.LogflarePostgres: env = append(env, - fmt.Sprintf("POSTGRES_BACKEND_URL=postgresql://%s:%s@%s:%d/%s", dbConfig.User, dbConfig.Password, dbConfig.Host, dbConfig.Port, dbConfig.Database), + fmt.Sprintf("POSTGRES_BACKEND_URL=postgresql://%s:%s@%s:%d/%s", dbConfig.User, dbConfig.Password, dbConfig.Host, dbConfig.Port, "_supabase"), "POSTGRES_BACKEND_SCHEMA=_analytics", ) } @@ -750,7 +750,7 @@ EOF fmt.Sprintf("DB_PORT=%d", dbConfig.Port), "DB_USER=supabase_admin", "DB_PASSWORD=" + dbConfig.Password, - "DB_NAME=" + dbConfig.Database, + "DB_NAME=_supabase", "DB_AFTER_CONNECT_QUERY=SET search_path TO _realtime", "DB_ENC_KEY=" + utils.Config.Realtime.EncryptionKey, "API_JWT_SECRET=" + utils.Config.Auth.JwtSecret, @@ -1045,7 +1045,7 @@ EOF "PORT=4000", fmt.Sprintf("PROXY_PORT_SESSION=%d", portSession), fmt.Sprintf("PROXY_PORT_TRANSACTION=%d", portTransaction), - fmt.Sprintf("DATABASE_URL=ecto://%s:%s@%s:%d/%s", dbConfig.User, dbConfig.Password, dbConfig.Host, dbConfig.Port, dbConfig.Database), + fmt.Sprintf("DATABASE_URL=ecto://%s:%s@%s:%d/%s", dbConfig.User, dbConfig.Password, dbConfig.Host, dbConfig.Port, "_supabase"), "CLUSTER_POSTGRES=true", "SECRET_KEY_BASE=" + utils.Config.Db.Pooler.SecretKeyBase, "VAULT_ENC_KEY=" + utils.Config.Db.Pooler.EncryptionKey, From 5e946870f2127d6e4bbf55edc0b4252f08f12148 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 24 Sep 2024 15:15:26 +0200 Subject: [PATCH 2/7] chore: fix tests mocks --- internal/db/reset/reset.go | 2 +- internal/db/reset/reset_test.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/db/reset/reset.go b/internal/db/reset/reset.go index 89a33fda5..d53392ae9 100644 --- a/internal/db/reset/reset.go +++ b/internal/db/reset/reset.go @@ -170,8 +170,8 @@ func recreateDatabase(ctx context.Context, options ...func(*pgx.ConnConfig)) err sql := migration.MigrationFile{ Statements: []string{ "DROP DATABASE IF EXISTS postgres WITH (FORCE)", - "DROP DATABASE IF EXISTS _supabase WITH (FORCE)", "CREATE DATABASE postgres WITH OWNER postgres", + "DROP DATABASE IF EXISTS _supabase WITH (FORCE)", "CREATE DATABASE _supabase WITH OWNER postgres", }, } diff --git a/internal/db/reset/reset_test.go b/internal/db/reset/reset_test.go index 03da968c5..ffb9898cf 100644 --- a/internal/db/reset/reset_test.go +++ b/internal/db/reset/reset_test.go @@ -145,6 +145,10 @@ func TestRecreateDatabase(t *testing.T) { Query("DROP DATABASE IF EXISTS postgres WITH (FORCE)"). Reply("DROP DATABASE"). Query("CREATE DATABASE postgres WITH OWNER postgres"). + Reply("CREATE DATABASE"). + Query("DROP DATABASE IF EXISTS _supabase WITH (FORCE)"). + Reply("DROP DATABASE"). + Query("CREATE DATABASE _supabase WITH OWNER postgres"). Reply("CREATE DATABASE") // Run test assert.NoError(t, recreateDatabase(context.Background(), conn.Intercept)) @@ -194,8 +198,11 @@ func TestRecreateDatabase(t *testing.T) { Reply("DO"). Query("DROP DATABASE IF EXISTS postgres WITH (FORCE)"). ReplyError(pgerrcode.ObjectInUse, `database "postgres" is used by an active logical replication slot`). - Query("CREATE DATABASE postgres WITH OWNER postgres") - // Run test + Query("CREATE DATABASE postgres WITH OWNER postgres"). + Query("DROP DATABASE IF EXISTS _supabase WITH (FORCE)"). + Reply("DROP DATABASE"). + Query("CREATE DATABASE _supabase WITH OWNER postgres"). + Reply("CREATE DATABASE") err := recreateDatabase(context.Background(), conn.Intercept) // Check error assert.ErrorContains(t, err, `ERROR: database "postgres" is used by an active logical replication slot (SQLSTATE 55006)`) From e6788f3bc03e91af979c000435cf6ba4610c3dcf Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 24 Sep 2024 18:03:41 +0200 Subject: [PATCH 3/7] chore: add realtime schema as well --- internal/db/start/templates/schema.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/db/start/templates/schema.sql b/internal/db/start/templates/schema.sql index 418f0ee05..45a936a2e 100644 --- a/internal/db/start/templates/schema.sql +++ b/internal/db/start/templates/schema.sql @@ -20,6 +20,8 @@ CREATE DATABASE _supabase WITH OWNER postgres; -- with non-user activity CREATE SCHEMA IF NOT EXISTS _realtime; ALTER SCHEMA _realtime OWNER TO postgres; +CREATE SCHEMA IF NOT EXISTS realtime; +ALTER SCHEMA realtime OWNER TO postgres; CREATE SCHEMA IF NOT EXISTS _analytics; ALTER SCHEMA _analytics OWNER TO postgres; From 76ef3d2d3ed394302a05bb67194aa30a0136ce87 Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 24 Sep 2024 19:25:36 +0200 Subject: [PATCH 4/7] fix: make the start work for postgres 14 and 13 --- internal/db/dump/dump.go | 4 ++++ internal/db/start/start.go | 11 +++++++++-- internal/db/start/templates/_supabase.sql | 18 ++++++++++++++++++ internal/db/start/templates/schema.sql | 19 ------------------- 4 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 internal/db/start/templates/_supabase.sql diff --git a/internal/db/dump/dump.go b/internal/db/dump/dump.go index e75cbb8b9..94e2ba7af 100644 --- a/internal/db/dump/dump.go +++ b/internal/db/dump/dump.go @@ -100,6 +100,10 @@ func dumpData(ctx context.Context, config pgconn.Config, schema, excludeTable [] // "storage", // "supabase_functions", "supabase_migrations", + // TODO: Remove in a few version in favor of _supabase internal db + "_analytics", + "_realtime", + "_supavisor", } var env []string if len(schema) > 0 { diff --git a/internal/db/start/start.go b/internal/db/start/start.go index c25aa24e1..8ed608ad7 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -29,6 +29,8 @@ var ( HealthTimeout = 120 * time.Second //go:embed templates/schema.sql initialSchema string + //go:embed templates/_supabase.sql + _supabaseSchema string ) func Run(ctx context.Context, fsys afero.Fs) error { @@ -82,7 +84,7 @@ func NewContainerConfig() container.Config { Retries: 3, }, Entrypoint: []string{"sh", "-c", `cat <<'EOF' > /etc/postgresql.schema.sql && cat <<'EOF' > /etc/postgresql-custom/pgsodium_root.key && docker-entrypoint.sh postgres -D /etc/postgresql -` + initialSchema + ` +` + initialSchema + "\n" + _supabaseSchema + "\n" + ` EOF ` + utils.Config.Db.RootKey + ` EOF @@ -122,7 +124,12 @@ func StartDatabase(ctx context.Context, fsys afero.Fs, w io.Writer, options ...f }, } if utils.Config.Db.MajorVersion <= 14 { - config.Entrypoint = nil + config.Entrypoint = []string{"sh", "-c", ` + cat <<'EOF' > /docker-entrypoint-initdb.d/supabase_schema.sql +` + _supabaseSchema + ` +EOF + docker-entrypoint.sh postgres -D /etc/postgresql + `} hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""} } // Creating volume will not override existing volume, so we must inspect explicitly diff --git a/internal/db/start/templates/_supabase.sql b/internal/db/start/templates/_supabase.sql new file mode 100644 index 000000000..d391166a2 --- /dev/null +++ b/internal/db/start/templates/_supabase.sql @@ -0,0 +1,18 @@ +CREATE DATABASE _supabase WITH OWNER postgres; + +-- Switch to the newly created _supabase database +\c _supabase +-- Create schemas in _supabase database for +-- internals tools and reports to not overload user database +-- with non-user activity +CREATE SCHEMA IF NOT EXISTS _realtime; +ALTER SCHEMA _realtime OWNER TO postgres; +CREATE SCHEMA IF NOT EXISTS realtime; +ALTER SCHEMA realtime OWNER TO postgres; + +CREATE SCHEMA IF NOT EXISTS _analytics; +ALTER SCHEMA _analytics OWNER TO postgres; + +CREATE SCHEMA IF NOT EXISTS _supavisor; +ALTER SCHEMA _supavisor OWNER TO postgres; +\c postgres \ No newline at end of file diff --git a/internal/db/start/templates/schema.sql b/internal/db/start/templates/schema.sql index 45a936a2e..2ff6cf48f 100644 --- a/internal/db/start/templates/schema.sql +++ b/internal/db/start/templates/schema.sql @@ -12,25 +12,6 @@ ALTER USER supabase_storage_admin WITH PASSWORD :'pgpass'; ALTER USER supabase_replication_admin WITH PASSWORD :'pgpass'; ALTER USER supabase_read_only_user WITH PASSWORD :'pgpass'; -CREATE DATABASE _supabase WITH OWNER postgres; --- Connect to the _supabase database -\c _supabase --- Create schemas in _supabase database for --- internals tools and reports to not overload user database --- with non-user activity -CREATE SCHEMA IF NOT EXISTS _realtime; -ALTER SCHEMA _realtime OWNER TO postgres; -CREATE SCHEMA IF NOT EXISTS realtime; -ALTER SCHEMA realtime OWNER TO postgres; - -CREATE SCHEMA IF NOT EXISTS _analytics; -ALTER SCHEMA _analytics OWNER TO postgres; - -CREATE SCHEMA IF NOT EXISTS _supavisor; -ALTER SCHEMA _supavisor OWNER TO postgres; --- Switch back to the main database -\c postgres - BEGIN; -- Create pg_net extension From ccf214d252a76ea87e5b3d72ed61c66c59ef7a48 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 27 Sep 2024 13:53:47 +0200 Subject: [PATCH 5/7] chore: restore realtime to postgres database --- internal/db/start/templates/_supabase.sql | 5 ----- internal/db/start/templates/schema.sql | 3 +++ internal/start/start.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/db/start/templates/_supabase.sql b/internal/db/start/templates/_supabase.sql index d391166a2..c339d2b61 100644 --- a/internal/db/start/templates/_supabase.sql +++ b/internal/db/start/templates/_supabase.sql @@ -5,11 +5,6 @@ CREATE DATABASE _supabase WITH OWNER postgres; -- Create schemas in _supabase database for -- internals tools and reports to not overload user database -- with non-user activity -CREATE SCHEMA IF NOT EXISTS _realtime; -ALTER SCHEMA _realtime OWNER TO postgres; -CREATE SCHEMA IF NOT EXISTS realtime; -ALTER SCHEMA realtime OWNER TO postgres; - CREATE SCHEMA IF NOT EXISTS _analytics; ALTER SCHEMA _analytics OWNER TO postgres; diff --git a/internal/db/start/templates/schema.sql b/internal/db/start/templates/schema.sql index 2ff6cf48f..534dd1207 100644 --- a/internal/db/start/templates/schema.sql +++ b/internal/db/start/templates/schema.sql @@ -12,6 +12,9 @@ ALTER USER supabase_storage_admin WITH PASSWORD :'pgpass'; ALTER USER supabase_replication_admin WITH PASSWORD :'pgpass'; ALTER USER supabase_read_only_user WITH PASSWORD :'pgpass'; +create schema if not exists _realtime; +alter schema _realtime owner to postgres; + BEGIN; -- Create pg_net extension diff --git a/internal/start/start.go b/internal/start/start.go index 413b8e2ed..72f34b8e7 100644 --- a/internal/start/start.go +++ b/internal/start/start.go @@ -750,7 +750,7 @@ EOF fmt.Sprintf("DB_PORT=%d", dbConfig.Port), "DB_USER=supabase_admin", "DB_PASSWORD=" + dbConfig.Password, - "DB_NAME=_supabase", + "DB_NAME=" + dbConfig.Database, "DB_AFTER_CONNECT_QUERY=SET search_path TO _realtime", "DB_ENC_KEY=" + utils.Config.Realtime.EncryptionKey, "API_JWT_SECRET=" + utils.Config.Auth.JwtSecret, From e911718c22eecae9b039119c289b766de58fe8fa Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 27 Sep 2024 16:06:33 +0200 Subject: [PATCH 6/7] chore: remove relatime migration --- internal/db/start/start.go | 2 +- internal/db/start/templates/_supabase.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/db/start/start.go b/internal/db/start/start.go index 8ed608ad7..9614528f5 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -235,7 +235,7 @@ func initRealtimeJob(host string) utils.DockerJob { "DB_PORT=5432", "DB_USER=supabase_admin", "DB_PASSWORD=" + utils.Config.Db.Password, - "DB_NAME=_supabase", + "DB_NAME=postgres", "DB_AFTER_CONNECT_QUERY=SET search_path TO _realtime", "DB_ENC_KEY=" + utils.Config.Realtime.EncryptionKey, "API_JWT_SECRET=" + utils.Config.Auth.JwtSecret, diff --git a/internal/db/start/templates/_supabase.sql b/internal/db/start/templates/_supabase.sql index c339d2b61..6e5d8487b 100644 --- a/internal/db/start/templates/_supabase.sql +++ b/internal/db/start/templates/_supabase.sql @@ -10,4 +10,4 @@ ALTER SCHEMA _analytics OWNER TO postgres; CREATE SCHEMA IF NOT EXISTS _supavisor; ALTER SCHEMA _supavisor OWNER TO postgres; -\c postgres \ No newline at end of file +\c postgres From 3115143937d6ed665bc547e66def7f96032b8c81 Mon Sep 17 00:00:00 2001 From: Andrew Valleteau Date: Fri, 27 Sep 2024 18:10:42 +0200 Subject: [PATCH 7/7] Update internal/db/start/start.go Co-authored-by: Han Qiao --- internal/db/start/start.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/db/start/start.go b/internal/db/start/start.go index 9614528f5..74e40554a 100644 --- a/internal/db/start/start.go +++ b/internal/db/start/start.go @@ -84,7 +84,8 @@ func NewContainerConfig() container.Config { Retries: 3, }, Entrypoint: []string{"sh", "-c", `cat <<'EOF' > /etc/postgresql.schema.sql && cat <<'EOF' > /etc/postgresql-custom/pgsodium_root.key && docker-entrypoint.sh postgres -D /etc/postgresql -` + initialSchema + "\n" + _supabaseSchema + "\n" + ` +` + initialSchema + ` +` + _supabaseSchema + ` EOF ` + utils.Config.Db.RootKey + ` EOF