-
Notifications
You must be signed in to change notification settings - Fork 384
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
PostgreSQL quota manager and storage backend (#3644)
This PR is based on Trillian's existing MySQL quota manager and storage backend. The first several commits were auto-generated by [this script](https://gist.github.com/robstradling/e6685c10534ca21bb10b2871c8a154c0), which forked the existing MySQL code into different directories (whilst preserving the git history) and then did a bunch of search'n'replacing to switch from the [database/sql](https://pkg.go.dev/database/sql) interface to the [jackc/pgx](https://pkg.go.dev/github.com/jackc/pgx/v5) interface. Improving performance is my main reason for using the pgx interface directly. In particular, the pgx interface has allowed me to use PostgreSQL's COPY interface for fast bulk-upserts. My motivations for putting together this PR are that (1) I and my colleagues at Sectigo have a fair amount of experience with PostgreSQL, but almost no experience with MySQL/MariaDB; and (2) we suffered a [CT log failure earlier this year](https://groups.google.com/a/chromium.org/g/ct-policy/c/038B7F4g8cU/m/KsOJaEhnBgAJ) due to MariaDB corruption after disk space exhaustion, and we are confident that PostgreSQL would not have broken under the same circumstances.
- Loading branch information
1 parent
d3a1031
commit baa721c
Showing
30 changed files
with
4,435 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| --- | ||
| name: Test PostgreSQL | ||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| pull_request: | ||
| workflow_dispatch: | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| jobs: | ||
| lint: | ||
| permissions: | ||
| contents: read # for actions/checkout to fetch code | ||
| pull-requests: read # for golangci/golangci-lint-action to fetch pull requests | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 | ||
|
|
||
| - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 | ||
| with: | ||
| go-version-file: go.mod | ||
| check-latest: true | ||
| cache: true | ||
|
|
||
| - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 | ||
| with: | ||
| version: 'v1.55.1' | ||
| args: ./storage/postgresql | ||
|
|
||
| integration-and-unit-tests: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 | ||
|
|
||
| - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 | ||
| with: | ||
| go-version-file: go.mod | ||
| check-latest: true | ||
| cache: true | ||
|
|
||
| - name: Build before tests | ||
| run: go mod download && go build ./... | ||
|
|
||
| - name: Run PostgreSQL | ||
| run: docker run --rm -d --name=pgsql -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust postgres:latest | ||
|
|
||
| - name: Wait for PostgreSQL | ||
| uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 | ||
| with: | ||
| timeout_seconds: 15 | ||
| max_attempts: 3 | ||
| retry_on: error | ||
| command: docker exec pgsql psql -U postgres -c "SELECT 1" | ||
|
|
||
| - name: Get PostgreSQL logs | ||
| run: docker logs pgsql 2>&1 | ||
|
|
||
| - name: Run integration tests | ||
| run: ./integration/integration_test.sh | ||
| env: | ||
| TEST_POSTGRESQL_URI: postgresql:///defaultdb?host=localhost&user=postgres&password=postgres | ||
| POSTGRESQL_IN_CONTAINER: true | ||
| POSTGRESQL_CONTAINER_NAME: pgsql | ||
|
|
||
| - name: Run unit tests | ||
| run: go test -v ./storage/postgresql/... ./quota/postgresqlqm/... | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,4 +11,5 @@ | |
| Antonio Marcedone <[email protected]> | ||
| Google LLC | ||
| Internet Security Research Group | ||
| Sectigo Limited | ||
| Vishal Kuo <[email protected]> | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -35,5 +35,6 @@ Paul Hadfield <[email protected]> <[email protected]> | |
| Pavel Kalinnikov <[email protected]> <[email protected]> | ||
| Pierre Phaneuf <[email protected]> <[email protected]> | ||
| Rob Percival <[email protected]> | ||
| Rob Stradling <[email protected]> | ||
| Roger Ng <[email protected]> <[email protected]> | ||
| Vishal Kuo <[email protected]> | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| // Copyright 2024 Trillian Authors. All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package postgresqlqm defines a PostgreSQL-based quota.Manager implementation. | ||
| package postgresqlqm | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
|
|
||
| "github.com/google/trillian/quota" | ||
| "github.com/jackc/pgx/v5/pgxpool" | ||
| ) | ||
|
|
||
| const ( | ||
| // DefaultMaxUnsequenced is a suggested value for MaxUnsequencedRows. | ||
| // Note that this is a Global/Write quota suggestion, so it applies across trees. | ||
| DefaultMaxUnsequenced = 500000 // About 2h of non-stop signing at 70QPS. | ||
|
|
||
| countFromExplainOutputQuery = "SELECT count_estimate($1)" | ||
| countFromUnsequencedQuery = "SELECT COUNT(*) FROM Unsequenced" | ||
| ) | ||
|
|
||
| // ErrTooManyUnsequencedRows is returned when tokens are requested but Unsequenced has grown | ||
| // beyond the configured limit. | ||
| var ErrTooManyUnsequencedRows = errors.New("too many unsequenced rows") | ||
|
|
||
| // QuotaManager is a PostgreSQL-based quota.Manager implementation. | ||
| // | ||
| // QuotaManager only implements Global/Write quotas, which is based on the number of Unsequenced | ||
| // rows (to be exact, tokens = MaxUnsequencedRows - actualUnsequencedRows). | ||
| // Other quotas are considered infinite. In other words, it attempts to protect the MMD SLO of all | ||
| // logs in the instance, but it does not make any attempt to ensure fairness, whether per-tree, | ||
| // per-intermediate-CA (in the case of Certificate Transparency), or any other dimension. | ||
| // | ||
| // It has two working modes: one estimates the number of Unsequenced rows by collecting information | ||
| // from EXPLAIN output; the other does a select count(*) on the Unsequenced table. Estimates are | ||
| // default, even though they are approximate, as they're constant time (select count(*) on | ||
| // PostgreSQL needs to traverse the index and may take quite a while to complete). | ||
| // Other estimation methods exist (see https://wiki.postgresql.org/wiki/Count_estimate), but using | ||
| // EXPLAIN output is the most accurate because it "fetches the actual current number of pages in | ||
| // the table (this is a cheap operation, not requiring a table scan). If that is different from | ||
| // relpages then reltuples is scaled accordingly to arrive at a current number-of-rows estimate." | ||
| // (quoting https://www.postgresql.org/docs/current/row-estimation-examples.html) | ||
| type QuotaManager struct { | ||
| DB *pgxpool.Pool | ||
| MaxUnsequencedRows int | ||
| UseSelectCount bool | ||
| } | ||
|
|
||
| // GetTokens implements quota.Manager.GetTokens. | ||
| // It doesn't actually reserve or retrieve tokens, instead it allows access based on the number of | ||
| // rows in the Unsequenced table. | ||
| func (m *QuotaManager) GetTokens(ctx context.Context, numTokens int, specs []quota.Spec) error { | ||
| for _, spec := range specs { | ||
| if spec.Group != quota.Global || spec.Kind != quota.Write { | ||
| continue | ||
| } | ||
| // Only allow global writes if Unsequenced is under the expected limit | ||
| count, err := m.countUnsequenced(ctx) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if count+numTokens > m.MaxUnsequencedRows { | ||
| return ErrTooManyUnsequencedRows | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // PutTokens implements quota.Manager.PutTokens. | ||
| // It's a noop for QuotaManager. | ||
| func (m *QuotaManager) PutTokens(ctx context.Context, numTokens int, specs []quota.Spec) error { | ||
| return nil | ||
| } | ||
|
|
||
| // ResetQuota implements quota.Manager.ResetQuota. | ||
| // It's a noop for QuotaManager. | ||
| func (m *QuotaManager) ResetQuota(ctx context.Context, specs []quota.Spec) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (m *QuotaManager) countUnsequenced(ctx context.Context) (int, error) { | ||
| if m.UseSelectCount { | ||
| return countFromTable(ctx, m.DB) | ||
| } | ||
| return countFromExplainOutput(ctx, m.DB) | ||
| } | ||
|
|
||
| func countFromExplainOutput(ctx context.Context, db *pgxpool.Pool) (int, error) { | ||
| var count int | ||
| if err := db.QueryRow(ctx, countFromExplainOutputQuery, "Unsequenced").Scan(&count); err != nil { | ||
| return 0, err | ||
| } | ||
| return count, nil | ||
| } | ||
|
|
||
| func countFromTable(ctx context.Context, db *pgxpool.Pool) (int, error) { | ||
| var count int | ||
| if err := db.QueryRow(ctx, countFromUnsequencedQuery).Scan(&count); err != nil { | ||
| return 0, err | ||
| } | ||
| return count, nil | ||
| } |
Oops, something went wrong.