Add production support for SQLite
This commit is contained in:
@@ -7,10 +7,10 @@ organised in private **vaults** (each vault is the web equivalent of one XCA
|
|||||||
database).
|
database).
|
||||||
|
|
||||||
- **`server/`** — Python + [kaya](https://github.com/woggioni/kaya) backend:
|
- **`server/`** — Python + [kaya](https://github.com/woggioni/kaya) backend:
|
||||||
OIDC login, Redis sessions, personal API tokens, Postgres (Tortoise ORM +
|
OIDC login, Redis sessions, personal API tokens, PostgreSQL or SQLite
|
||||||
aerich migrations), AES-256-GCM private-key encryption, and a complete REST
|
(Tortoise ORM + aerich migrations), AES-256-GCM private-key encryption, and
|
||||||
API. The pure PKI domain logic lives in `packages/kca-core`; the deployable
|
a complete REST API. The pure PKI domain logic lives in `packages/kca-core`;
|
||||||
app is `packages/kca-app`.
|
the deployable app is `packages/kca-app`.
|
||||||
- **`web/`** — Rust + [Sycamore](https://sycamore.dev) compiled to WebAssembly
|
- **`web/`** — Rust + [Sycamore](https://sycamore.dev) compiled to WebAssembly
|
||||||
by [Trunk](https://trunkrs.dev), styled with Bootstrap 5.3 (vendored under
|
by [Trunk](https://trunkrs.dev), styled with Bootstrap 5.3 (vendored under
|
||||||
`web/vendor/`) and a dark PKI-console theme.
|
`web/vendor/`) and a dark PKI-console theme.
|
||||||
@@ -50,6 +50,45 @@ OIDC_0_REDIRECT_URI=https://kca.example.com/auth/default/callback
|
|||||||
|
|
||||||
`server/.env.example` documents every setting.
|
`server/.env.example` documents every setting.
|
||||||
|
|
||||||
|
## Database backends
|
||||||
|
|
||||||
|
PostgreSQL is the default production backend. KCA also supports persistent,
|
||||||
|
file-backed SQLite for single-writer deployments. Configure it from `server/`
|
||||||
|
with either the component settings:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export DATABASE_ENGINE=sqlite
|
||||||
|
export DATABASE_NAME=/data/kca.db
|
||||||
|
export DATABASE_OPTIONS=busy_timeout=5000
|
||||||
|
```
|
||||||
|
|
||||||
|
or one URL:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export DATABASE_URL='sqlite:///data/kca.db?busy_timeout=5000'
|
||||||
|
```
|
||||||
|
|
||||||
|
Run migrations before starting or upgrading the app:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
uv run aerich upgrade
|
||||||
|
```
|
||||||
|
|
||||||
|
Tortoise enables WAL journaling and foreign-key enforcement by default; the
|
||||||
|
busy timeout allows concurrent readers and a writer to wait briefly for locks.
|
||||||
|
Only in-memory SQLite test databases create their schemas implicitly.
|
||||||
|
|
||||||
|
For containers, mount a persistent volume at `/data` for both the migration
|
||||||
|
and application containers and use the same SQLite URL for each. The image's
|
||||||
|
`/data` directory is writable by its non-root application user. When invoking
|
||||||
|
`aerich upgrade` directly in the image, set its working directory to `/app`
|
||||||
|
(for example, `docker run -w /app ...`). Keep SQLite to one application
|
||||||
|
writer/replica, and use SQLite-aware backups rather than independently copying
|
||||||
|
live database, WAL and shared-memory files.
|
||||||
|
|
||||||
|
When changing models, migration SQL must support both PostgreSQL and SQLite.
|
||||||
|
The PostgreSQL migration commands below remain the default development flow.
|
||||||
|
|
||||||
## What it does
|
## What it does
|
||||||
|
|
||||||
### Vaults
|
### Vaults
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# Environment configuration for KCA. Copy to .env or export the variables.
|
# Environment configuration for KCA. Copy to .env or export the variables.
|
||||||
|
|
||||||
# --- Database --------------------------------------------------------------
|
# --- Database --------------------------------------------------------------
|
||||||
|
# PostgreSQL (default). Run `aerich upgrade` before starting the app.
|
||||||
DATABASE_ENGINE=postgres
|
DATABASE_ENGINE=postgres
|
||||||
DATABASE_HOST=localhost
|
DATABASE_HOST=localhost
|
||||||
DATABASE_PORT=5432
|
DATABASE_PORT=5432
|
||||||
@@ -10,6 +11,15 @@ DATABASE_PASSWORD=password
|
|||||||
#DATABASE_OPTIONS=
|
#DATABASE_OPTIONS=
|
||||||
#DATABASE_URL=
|
#DATABASE_URL=
|
||||||
|
|
||||||
|
# SQLite production alternative: DATABASE_NAME is an absolute or relative file
|
||||||
|
# path; host, port, user and password are ignored. Use a persistent writable
|
||||||
|
# filesystem and run `aerich upgrade` before starting the app.
|
||||||
|
#DATABASE_ENGINE=sqlite
|
||||||
|
#DATABASE_NAME=/data/kca.db
|
||||||
|
#DATABASE_OPTIONS=busy_timeout=5000
|
||||||
|
# Equivalent single-setting form:
|
||||||
|
#DATABASE_URL=sqlite:///data/kca.db?busy_timeout=5000
|
||||||
|
|
||||||
# --- OIDC ------------------------------------------------------------------
|
# --- OIDC ------------------------------------------------------------------
|
||||||
# Indexed blocks OIDC_0_*, OIDC_1_*, ... with contiguous indices from 0.
|
# Indexed blocks OIDC_0_*, OIDC_1_*, ... with contiguous indices from 0.
|
||||||
OIDC_0_PROVIDER=default
|
OIDC_0_PROVIDER=default
|
||||||
|
|||||||
+2
-1
@@ -67,7 +67,8 @@ RUN sed -i 's|http://deb.debian.org|https://deb.debian.org|g' \
|
|||||||
&& apt-get install -y --no-install-recommends ca-certificates tzdata libssl3 \
|
&& apt-get install -y --no-install-recommends ca-certificates tzdata libssl3 \
|
||||||
&& rm -rf /var/lib/apt/lists/* \
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
&& groupadd --system app \
|
&& groupadd --system app \
|
||||||
&& useradd --system --gid app --no-create-home --shell /usr/sbin/nologin app
|
&& useradd --system --gid app --no-create-home --shell /usr/sbin/nologin app \
|
||||||
|
&& install -d -o app -g app /data
|
||||||
|
|
||||||
COPY --from=builder /opt/venv /opt/venv
|
COPY --from=builder /opt/venv /opt/venv
|
||||||
COPY --from=builder /repo/server/migrations /app/migrations
|
COPY --from=builder /repo/server/migrations /app/migrations
|
||||||
|
|||||||
@@ -3,7 +3,223 @@ from tortoise import BaseDBAsyncClient
|
|||||||
RUN_IN_TRANSACTION = True
|
RUN_IN_TRANSACTION = True
|
||||||
|
|
||||||
|
|
||||||
|
# SQLite production support was introduced after migration 4. Bootstrap new
|
||||||
|
# SQLite databases directly at that current schema; migrations 1-4 are no-ops
|
||||||
|
# on SQLite. PostgreSQL retains the historical migration path below.
|
||||||
|
SQLITE_UPGRADE = """
|
||||||
|
CREATE TABLE IF NOT EXISTS "api_token" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"user_issuer" VARCHAR(2048) NOT NULL,
|
||||||
|
"user_sub" VARCHAR(255) NOT NULL,
|
||||||
|
"name" VARCHAR(100) NOT NULL,
|
||||||
|
"token_prefix" VARCHAR(16) NOT NULL,
|
||||||
|
"token_hash" VARCHAR(64) NOT NULL UNIQUE,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"last_used_at" TIMESTAMP,
|
||||||
|
"expires_at" DATE
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_api_token_user_is_612f0d" ON "api_token" ("user_issuer", "user_sub");
|
||||||
|
CREATE TABLE IF NOT EXISTS "ocsp_snapshot" (
|
||||||
|
"ca_id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"dirty" INT NOT NULL,
|
||||||
|
"published_at" TIMESTAMP,
|
||||||
|
"last_error" TEXT,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_ocsp_snapsh_dirty_a8d440" ON "ocsp_snapshot" ("dirty");
|
||||||
|
CREATE TABLE IF NOT EXISTS "app_user" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"issuer" VARCHAR(2048) NOT NULL,
|
||||||
|
"sub" VARCHAR(255) NOT NULL,
|
||||||
|
"display_name" VARCHAR(200) NOT NULL,
|
||||||
|
"name" VARCHAR(255),
|
||||||
|
"family_name" VARCHAR(255),
|
||||||
|
"given_name" VARCHAR(255),
|
||||||
|
"middle_name" VARCHAR(255),
|
||||||
|
"nickname" VARCHAR(255),
|
||||||
|
"preferred_username" VARCHAR(255),
|
||||||
|
"profile" VARCHAR(2048),
|
||||||
|
"picture" VARCHAR(2048),
|
||||||
|
"website" VARCHAR(2048),
|
||||||
|
"gender" VARCHAR(100),
|
||||||
|
"birthdate" VARCHAR(32),
|
||||||
|
"zoneinfo" VARCHAR(100),
|
||||||
|
"locale" VARCHAR(100),
|
||||||
|
"updated_at" BIGINT,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"last_seen_at" TIMESTAMP NOT NULL,
|
||||||
|
CONSTRAINT "uid_app_user_issuer_d243fe" UNIQUE ("issuer", "sub")
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS "vault" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"owner_issuer" VARCHAR(2048) NOT NULL,
|
||||||
|
"owner_sub" VARCHAR(255) NOT NULL,
|
||||||
|
"name" VARCHAR(200) NOT NULL,
|
||||||
|
"description" TEXT NOT NULL,
|
||||||
|
"settings" JSON NOT NULL,
|
||||||
|
"version" INT NOT NULL,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_vault_owner_i_a035fb" ON "vault" ("owner_issuer", "owner_sub");
|
||||||
|
CREATE TABLE IF NOT EXISTS "key" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"name" VARCHAR(200) NOT NULL,
|
||||||
|
"comment" TEXT NOT NULL,
|
||||||
|
"source" VARCHAR(20) NOT NULL,
|
||||||
|
"key_type" VARCHAR(20) NOT NULL,
|
||||||
|
"key_size" INT,
|
||||||
|
"curve" VARCHAR(100),
|
||||||
|
"public_der" BLOB NOT NULL,
|
||||||
|
"public_key_sha256" VARCHAR(64) NOT NULL,
|
||||||
|
"private_der" BLOB,
|
||||||
|
"private_nonce" BLOB,
|
||||||
|
"has_private" INT NOT NULL,
|
||||||
|
"version" INT NOT NULL,
|
||||||
|
"search_text" TEXT NOT NULL,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL,
|
||||||
|
"vault_id" CHAR(36) NOT NULL REFERENCES "vault" ("id") ON DELETE CASCADE,
|
||||||
|
CONSTRAINT "uid_key_vault_i_e0ffb5" UNIQUE ("vault_id", "public_key_sha256")
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_key_public__eba47e" ON "key" ("public_key_sha256");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_key_vault_i_a6f27f" ON "key" ("vault_id", "name");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_key_vault_i_c2e997" ON "key" ("vault_id", "key_type");
|
||||||
|
CREATE TABLE IF NOT EXISTS "request" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"name" VARCHAR(200) NOT NULL,
|
||||||
|
"comment" TEXT NOT NULL,
|
||||||
|
"source" VARCHAR(20) NOT NULL,
|
||||||
|
"subject_der" BLOB NOT NULL,
|
||||||
|
"req_der" BLOB NOT NULL,
|
||||||
|
"public_key_der" BLOB NOT NULL,
|
||||||
|
"request_sha256" VARCHAR(64) NOT NULL,
|
||||||
|
"signed" INT NOT NULL,
|
||||||
|
"signed_at" TIMESTAMP,
|
||||||
|
"attributes" JSON NOT NULL,
|
||||||
|
"version" INT NOT NULL,
|
||||||
|
"search_text" TEXT NOT NULL,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL,
|
||||||
|
"key_id" CHAR(36) REFERENCES "key" ("id") ON DELETE SET NULL,
|
||||||
|
"vault_id" CHAR(36) NOT NULL REFERENCES "vault" ("id") ON DELETE CASCADE,
|
||||||
|
CONSTRAINT "uid_request_vault_i_b661f4" UNIQUE ("vault_id", "request_sha256")
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_request_request_fad061" ON "request" ("request_sha256");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_request_vault_i_346e22" ON "request" ("vault_id", "name");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_request_vault_i_2cd9ef" ON "request" ("vault_id", "signed");
|
||||||
|
CREATE TABLE IF NOT EXISTS "certificate" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"name" VARCHAR(200) NOT NULL,
|
||||||
|
"comment" TEXT NOT NULL,
|
||||||
|
"source" VARCHAR(20) NOT NULL,
|
||||||
|
"serial_hex" VARCHAR(128) NOT NULL,
|
||||||
|
"subject_der" BLOB NOT NULL,
|
||||||
|
"issuer_der" BLOB NOT NULL,
|
||||||
|
"subject_hash" VARCHAR(64) NOT NULL,
|
||||||
|
"issuer_hash" VARCHAR(64) NOT NULL,
|
||||||
|
"not_before" TIMESTAMP NOT NULL,
|
||||||
|
"not_after" TIMESTAMP NOT NULL,
|
||||||
|
"no_expiration" INT NOT NULL,
|
||||||
|
"is_ca" INT NOT NULL,
|
||||||
|
"path_length" INT,
|
||||||
|
"cert_der" BLOB NOT NULL,
|
||||||
|
"public_key_der" BLOB NOT NULL,
|
||||||
|
"signature_algorithm" VARCHAR(100) NOT NULL,
|
||||||
|
"fingerprint_md5" VARCHAR(32) NOT NULL,
|
||||||
|
"fingerprint_sha1" VARCHAR(40) NOT NULL,
|
||||||
|
"fingerprint_sha256" VARCHAR(64) NOT NULL,
|
||||||
|
"version" INT NOT NULL,
|
||||||
|
"search_text" TEXT NOT NULL,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL,
|
||||||
|
"issuer_id" CHAR(36) REFERENCES "certificate" ("id") ON DELETE SET NULL,
|
||||||
|
"key_id" CHAR(36) REFERENCES "key" ("id") ON DELETE SET NULL,
|
||||||
|
"request_id" CHAR(36) REFERENCES "request" ("id") ON DELETE SET NULL,
|
||||||
|
"vault_id" CHAR(36) NOT NULL REFERENCES "vault" ("id") ON DELETE CASCADE,
|
||||||
|
CONSTRAINT "uid_certificate_vault_i_842384" UNIQUE ("vault_id", "fingerprint_sha256")
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_serial__787fd1" ON "certificate" ("serial_hex");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_subject_ed411b" ON "certificate" ("subject_hash");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_issuer__b8daa0" ON "certificate" ("issuer_hash");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_is_ca_906cf3" ON "certificate" ("is_ca");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_fingerp_12be2c" ON "certificate" ("fingerprint_sha256");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_vault_i_18b9a0" ON "certificate" ("vault_id", "name");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_vault_i_dfb18f" ON "certificate" ("vault_id", "issuer_id");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_vault_i_eded8f" ON "certificate" ("vault_id", "not_after");
|
||||||
|
CREATE TABLE IF NOT EXISTS "certificate_revocation" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"serial_hex" VARCHAR(128) NOT NULL,
|
||||||
|
"revoked_at" TIMESTAMP NOT NULL,
|
||||||
|
"invalidity_date" TIMESTAMP,
|
||||||
|
"reason" VARCHAR(50) NOT NULL,
|
||||||
|
"crl_number" INT NOT NULL,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"ca_id" CHAR(36) NOT NULL REFERENCES "certificate" ("id") ON DELETE CASCADE,
|
||||||
|
CONSTRAINT "uid_certificate_ca_id_1c9aa5" UNIQUE ("ca_id", "serial_hex")
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_certificate_serial__8fa20f" ON "certificate_revocation" ("serial_hex");
|
||||||
|
CREATE TABLE IF NOT EXISTS "crl" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"name" VARCHAR(200) NOT NULL,
|
||||||
|
"comment" TEXT NOT NULL,
|
||||||
|
"source" VARCHAR(20) NOT NULL,
|
||||||
|
"crl_der" BLOB NOT NULL,
|
||||||
|
"crl_hash" VARCHAR(64) NOT NULL,
|
||||||
|
"crl_number" INT,
|
||||||
|
"last_update" TIMESTAMP NOT NULL,
|
||||||
|
"next_update" TIMESTAMP,
|
||||||
|
"revoked_count" INT NOT NULL,
|
||||||
|
"signature_algorithm" VARCHAR(100) NOT NULL,
|
||||||
|
"search_text" TEXT NOT NULL,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL,
|
||||||
|
"issuer_id" CHAR(36) REFERENCES "certificate" ("id") ON DELETE SET NULL,
|
||||||
|
"vault_id" CHAR(36) NOT NULL REFERENCES "vault" ("id") ON DELETE CASCADE,
|
||||||
|
CONSTRAINT "uid_crl_vault_i_e5461a" UNIQUE ("vault_id", "crl_hash")
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_crl_crl_has_c44d61" ON "crl" ("crl_hash");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_crl_vault_i_6af15c" ON "crl" ("vault_id", "name");
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_crl_vault_i_576344" ON "crl" ("vault_id", "issuer_id");
|
||||||
|
CREATE TABLE IF NOT EXISTS "template" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"name" VARCHAR(200) NOT NULL,
|
||||||
|
"comment" TEXT NOT NULL,
|
||||||
|
"source" VARCHAR(20) NOT NULL,
|
||||||
|
"subject" JSON NOT NULL,
|
||||||
|
"settings" JSON NOT NULL,
|
||||||
|
"version" INT NOT NULL,
|
||||||
|
"search_text" TEXT NOT NULL,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL,
|
||||||
|
"vault_id" CHAR(36) NOT NULL REFERENCES "vault" ("id") ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS "idx_template_vault_i_18d8f5" ON "template" ("vault_id", "name");
|
||||||
|
CREATE TABLE IF NOT EXISTS "authority" (
|
||||||
|
"id" CHAR(36) NOT NULL PRIMARY KEY,
|
||||||
|
"template_key" VARCHAR(64),
|
||||||
|
"crl_expire" TIMESTAMP,
|
||||||
|
"crl_number" INT NOT NULL,
|
||||||
|
"crl_days" INT NOT NULL,
|
||||||
|
"tls_auth_key" BLOB,
|
||||||
|
"tls_auth_key_nonce" BLOB,
|
||||||
|
"created_at" TIMESTAMP NOT NULL,
|
||||||
|
"updated_at" TIMESTAMP NOT NULL,
|
||||||
|
"template_id" CHAR(36) REFERENCES "template" ("id") ON DELETE SET NULL,
|
||||||
|
"certificate_id" CHAR(36) NOT NULL UNIQUE REFERENCES "certificate" ("id") ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS "aerich" (
|
||||||
|
"id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
|
||||||
|
"version" VARCHAR(255) NOT NULL,
|
||||||
|
"app" VARCHAR(100) NOT NULL,
|
||||||
|
"content" JSON NOT NULL
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
async def upgrade(db: BaseDBAsyncClient) -> str:
|
async def upgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return SQLITE_UPGRADE
|
||||||
return """
|
return """
|
||||||
CREATE TABLE IF NOT EXISTS "api_token" (
|
CREATE TABLE IF NOT EXISTS "api_token" (
|
||||||
"id" UUID NOT NULL PRIMARY KEY,
|
"id" UUID NOT NULL PRIMARY KEY,
|
||||||
|
|||||||
@@ -4,11 +4,16 @@ RUN_IN_TRANSACTION = True
|
|||||||
|
|
||||||
|
|
||||||
async def upgrade(db: BaseDBAsyncClient) -> str:
|
async def upgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
# Migration 0 bootstraps SQLite at the current schema.
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
ALTER TABLE "certificate" ADD "no_expiration" BOOL NOT NULL DEFAULT False;"""
|
ALTER TABLE "certificate" ADD "no_expiration" BOOL NOT NULL DEFAULT False;"""
|
||||||
|
|
||||||
|
|
||||||
async def downgrade(db: BaseDBAsyncClient) -> str:
|
async def downgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
ALTER TABLE "certificate" DROP COLUMN "no_expiration";"""
|
ALTER TABLE "certificate" DROP COLUMN "no_expiration";"""
|
||||||
|
|
||||||
|
|||||||
@@ -4,11 +4,16 @@ RUN_IN_TRANSACTION = True
|
|||||||
|
|
||||||
|
|
||||||
async def upgrade(db: BaseDBAsyncClient) -> str:
|
async def upgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
# Migration 0 bootstraps SQLite at the current schema.
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
ALTER TABLE "authority" ADD "template_key" VARCHAR(64);"""
|
ALTER TABLE "authority" ADD "template_key" VARCHAR(64);"""
|
||||||
|
|
||||||
|
|
||||||
async def downgrade(db: BaseDBAsyncClient) -> str:
|
async def downgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
ALTER TABLE "authority" DROP COLUMN "template_key";"""
|
ALTER TABLE "authority" DROP COLUMN "template_key";"""
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,9 @@ RUN_IN_TRANSACTION = True
|
|||||||
|
|
||||||
|
|
||||||
async def upgrade(db: BaseDBAsyncClient) -> str:
|
async def upgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
# Migration 0 bootstraps SQLite at the current schema.
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
CREATE TABLE IF NOT EXISTS "ocsp_snapshot" (
|
CREATE TABLE IF NOT EXISTS "ocsp_snapshot" (
|
||||||
"ca_id" UUID NOT NULL PRIMARY KEY,
|
"ca_id" UUID NOT NULL PRIMARY KEY,
|
||||||
@@ -17,6 +20,8 @@ COMMENT ON TABLE "ocsp_snapshot" IS 'Publication state for one CA''s OCSP status
|
|||||||
|
|
||||||
|
|
||||||
async def downgrade(db: BaseDBAsyncClient) -> str:
|
async def downgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
DROP TABLE IF EXISTS "ocsp_snapshot";"""
|
DROP TABLE IF EXISTS "ocsp_snapshot";"""
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,9 @@ RUN_IN_TRANSACTION = True
|
|||||||
|
|
||||||
|
|
||||||
async def upgrade(db: BaseDBAsyncClient) -> str:
|
async def upgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
# Migration 0 bootstraps SQLite at the current schema.
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
DROP INDEX IF EXISTS "idx_vault_owner_s_cdc1f8";
|
DROP INDEX IF EXISTS "idx_vault_owner_s_cdc1f8";
|
||||||
ALTER TABLE "app_user" DROP CONSTRAINT IF EXISTS "app_user_sub_key";
|
ALTER TABLE "app_user" DROP CONSTRAINT IF EXISTS "app_user_sub_key";
|
||||||
@@ -21,6 +24,8 @@ async def upgrade(db: BaseDBAsyncClient) -> str:
|
|||||||
|
|
||||||
|
|
||||||
async def downgrade(db: BaseDBAsyncClient) -> str:
|
async def downgrade(db: BaseDBAsyncClient) -> str:
|
||||||
|
if db.capabilities.dialect == "sqlite":
|
||||||
|
return "SELECT 1;"
|
||||||
return """
|
return """
|
||||||
DROP INDEX IF EXISTS "idx_vault_owner_i_a035fb";
|
DROP INDEX IF EXISTS "idx_vault_owner_i_a035fb";
|
||||||
DROP INDEX IF EXISTS "uid_app_user_issuer_d243fe";
|
DROP INDEX IF EXISTS "uid_app_user_issuer_d243fe";
|
||||||
|
|||||||
@@ -142,13 +142,18 @@ def _database_url_from_parts(
|
|||||||
name: str,
|
name: str,
|
||||||
options: str,
|
options: str,
|
||||||
) -> str:
|
) -> str:
|
||||||
netloc = quote(user, safe="")
|
if engine == "sqlite":
|
||||||
if password:
|
# DATABASE_NAME is a file path for SQLite (or :memory: for tests), so
|
||||||
netloc += ":" + quote(password, safe="")
|
# network credentials and host/port settings do not apply.
|
||||||
netloc += "@" + host
|
url = f"sqlite://{name}"
|
||||||
if port:
|
else:
|
||||||
netloc += ":" + port
|
netloc = quote(user, safe="")
|
||||||
url = f"{engine}://{netloc}/{name}"
|
if password:
|
||||||
|
netloc += ":" + quote(password, safe="")
|
||||||
|
netloc += "@" + host
|
||||||
|
if port:
|
||||||
|
netloc += ":" + port
|
||||||
|
url = f"{engine}://{netloc}/{name}"
|
||||||
options = options.lstrip("?")
|
options = options.lstrip("?")
|
||||||
if options:
|
if options:
|
||||||
url += "?" + options
|
url += "?" + options
|
||||||
|
|||||||
@@ -23,10 +23,10 @@ It deliberately avoids the global-fallback singleton
|
|||||||
per process and would therefore break across event loops.
|
per process and would therefore break across event loops.
|
||||||
|
|
||||||
Schema management is split by backend: in-memory sqlite databases (the
|
Schema management is split by backend: in-memory sqlite databases (the
|
||||||
test suite) get ``generate_schemas`` on every fresh context; Postgres
|
test suite) get ``generate_schemas`` on every fresh context; Postgres and
|
||||||
schemas are owned by aerich migrations and must be applied externally
|
file-backed SQLite schemas are owned by aerich migrations and must be
|
||||||
(``aerich upgrade``, run by the ``db-migrate`` compose service) before
|
applied externally (``aerich upgrade``, run by the ``db-migrate`` compose
|
||||||
the app serves requests.
|
service) before the app serves requests.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -104,12 +104,17 @@ class TortoiseMixin(KayaMixin):
|
|||||||
modules={"models": self._models_modules},
|
modules={"models": self._models_modules},
|
||||||
)
|
)
|
||||||
log.info("database context initialized (%s)", _safe_url(self._database_url))
|
log.info("database context initialized (%s)", _safe_url(self._database_url))
|
||||||
# Schema creation is only done for sqlite (in-memory test
|
# Only ephemeral test databases are created implicitly. Postgres
|
||||||
# databases). Postgres schemas are managed by aerich migrations
|
# and persistent SQLite schemas are managed by aerich migrations
|
||||||
# (applied by the db-migrate compose service / `aerich upgrade`).
|
# (applied by the db-migrate compose service / `aerich upgrade`).
|
||||||
if self._database_url.startswith("sqlite"):
|
connection = ctx.connections.get("default")
|
||||||
|
is_memory_sqlite = (
|
||||||
|
connection.capabilities.dialect == "sqlite"
|
||||||
|
and getattr(connection, "filename", None) == ":memory:"
|
||||||
|
)
|
||||||
|
if is_memory_sqlite:
|
||||||
await ctx.generate_schemas()
|
await ctx.generate_schemas()
|
||||||
log.info("sqlite schemas generated")
|
log.info("in-memory sqlite schemas generated")
|
||||||
return ctx
|
return ctx
|
||||||
|
|
||||||
async def ensure_context(self) -> None:
|
async def ensure_context(self) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""Persistent SQLite configuration and migration integration tests."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from contextlib import closing
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from kca.config import _database_url_from_parts
|
||||||
|
from kca.models import Vault
|
||||||
|
from kca.tortoise_mixin import TortoiseMixin
|
||||||
|
from tortoise.context import TortoiseContext
|
||||||
|
|
||||||
|
SERVER_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
MIGRATIONS_DIR = SERVER_ROOT / "migrations" / "models"
|
||||||
|
|
||||||
|
|
||||||
|
class SqliteConfigurationTests(unittest.TestCase):
|
||||||
|
def test_database_url_from_sqlite_parts(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
_database_url_from_parts(
|
||||||
|
"sqlite", "ignored", "ignored", "ignored", "5432", "/data/kca.db", "busy_timeout=5000"
|
||||||
|
),
|
||||||
|
"sqlite:///data/kca.db?busy_timeout=5000",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
_database_url_from_parts("sqlite", "", None, "", "", "data/kca.db", ""),
|
||||||
|
"sqlite://data/kca.db",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
_database_url_from_parts("sqlite", "", None, "", "", ":memory:", ""),
|
||||||
|
"sqlite://:memory:",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SqliteMemoryTests(unittest.TestCase):
|
||||||
|
def test_memory_url_with_options_generates_schemas(self) -> None:
|
||||||
|
async def create_vault() -> None:
|
||||||
|
mixin = TortoiseMixin("sqlite://:memory:?busy_timeout=5000", ["kca.models"])
|
||||||
|
await mixin.ensure_context()
|
||||||
|
try:
|
||||||
|
await Vault.create(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
owner_issuer="https://issuer.example",
|
||||||
|
owner_sub="memory-user",
|
||||||
|
name="Memory vault",
|
||||||
|
settings={},
|
||||||
|
)
|
||||||
|
self.assertEqual(await Vault.all().count(), 1)
|
||||||
|
finally:
|
||||||
|
await mixin.aclose()
|
||||||
|
|
||||||
|
asyncio.run(create_vault())
|
||||||
|
|
||||||
|
|
||||||
|
class SqliteFileMigrationTests(unittest.TestCase):
|
||||||
|
def test_migrations_pragmas_and_persistence(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db_path = Path(tmp) / "kca.db"
|
||||||
|
database_url = f"sqlite://{db_path}?busy_timeout=5000"
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["DATABASE_URL"] = database_url
|
||||||
|
aerich = Path(sys.executable).with_name("aerich")
|
||||||
|
|
||||||
|
for _ in range(2):
|
||||||
|
result = subprocess.run(
|
||||||
|
[str(aerich), "upgrade"],
|
||||||
|
cwd=SERVER_ROOT,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||||
|
|
||||||
|
expected_tables = {
|
||||||
|
"aerich",
|
||||||
|
"api_token",
|
||||||
|
"app_user",
|
||||||
|
"authority",
|
||||||
|
"certificate",
|
||||||
|
"certificate_revocation",
|
||||||
|
"crl",
|
||||||
|
"key",
|
||||||
|
"ocsp_snapshot",
|
||||||
|
"request",
|
||||||
|
"template",
|
||||||
|
"vault",
|
||||||
|
}
|
||||||
|
migration_count = len(list(MIGRATIONS_DIR.glob("[0-9]_*.py")))
|
||||||
|
with closing(sqlite3.connect(db_path)) as db:
|
||||||
|
tables = {
|
||||||
|
row[0]
|
||||||
|
for row in db.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
self.assertEqual(tables, expected_tables)
|
||||||
|
self.assertEqual(db.execute("SELECT COUNT(*) FROM aerich").fetchone()[0], migration_count)
|
||||||
|
certificate_columns = {row[1] for row in db.execute("PRAGMA table_info(certificate)")}
|
||||||
|
authority_columns = {row[1] for row in db.execute("PRAGMA table_info(authority)")}
|
||||||
|
user_columns = {row[1] for row in db.execute("PRAGMA table_info(app_user)")}
|
||||||
|
self.assertIn("no_expiration", certificate_columns)
|
||||||
|
self.assertIn("template_key", authority_columns)
|
||||||
|
self.assertIn("issuer", user_columns)
|
||||||
|
|
||||||
|
vault_id = uuid.uuid4()
|
||||||
|
|
||||||
|
async def write_vault() -> None:
|
||||||
|
ctx = TortoiseContext()
|
||||||
|
with ctx:
|
||||||
|
await ctx.init(db_url=database_url, modules={"models": ["kca.models"]})
|
||||||
|
try:
|
||||||
|
connection = ctx.connections.get("default")
|
||||||
|
foreign_keys = await connection.execute_query_dict("PRAGMA foreign_keys")
|
||||||
|
journal_mode = await connection.execute_query_dict("PRAGMA journal_mode")
|
||||||
|
busy_timeout = await connection.execute_query_dict("PRAGMA busy_timeout")
|
||||||
|
self.assertEqual(foreign_keys[0]["foreign_keys"], 1)
|
||||||
|
self.assertEqual(journal_mode[0]["journal_mode"].lower(), "wal")
|
||||||
|
self.assertEqual(busy_timeout[0]["timeout"], 5000)
|
||||||
|
await Vault.create(
|
||||||
|
id=vault_id,
|
||||||
|
owner_issuer="https://issuer.example",
|
||||||
|
owner_sub="sqlite-user",
|
||||||
|
name="Persistent vault",
|
||||||
|
settings={},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await ctx.close_connections()
|
||||||
|
|
||||||
|
asyncio.run(write_vault())
|
||||||
|
with closing(sqlite3.connect(db_path)) as db:
|
||||||
|
row = db.execute("SELECT name FROM vault WHERE id = ?", (str(vault_id),)).fetchone()
|
||||||
|
self.assertEqual(row, ("Persistent vault",))
|
||||||
|
|
||||||
|
# SQLite bootstraps at the current schema, so historical downgrades
|
||||||
|
# are executable no-ops and preserve both the schema and its data.
|
||||||
|
for _ in range(migration_count - 1):
|
||||||
|
result = subprocess.run(
|
||||||
|
[str(aerich), "downgrade", "--yes"],
|
||||||
|
cwd=SERVER_ROOT,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||||
|
result = subprocess.run(
|
||||||
|
[str(aerich), "upgrade"],
|
||||||
|
cwd=SERVER_ROOT,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||||
|
with closing(sqlite3.connect(db_path)) as db:
|
||||||
|
self.assertEqual(db.execute("SELECT COUNT(*) FROM aerich").fetchone()[0], migration_count)
|
||||||
|
row = db.execute("SELECT name FROM vault WHERE id = ?", (str(vault_id),)).fetchone()
|
||||||
|
self.assertEqual(row, ("Persistent vault",))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user