Merge pull request #110 from authzed/materialize-api

introduces the gRPC API for Authzed Materialize

.github/workflows/lint.yaml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: "actions/checkout@v3"
       - uses: "authzed/actions/yaml-lint@main"
-      - uses: "bufbuild/buf-setup-action@v1.30.0"
+      - uses: "bufbuild/buf-setup-action@v1.32.2"
         with:
           version: "1.30.0"
       - uses: "bufbuild/buf-lint-action@v1"

.github/workflows/release.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v3"
-      - uses: "bufbuild/buf-setup-action@v1"
+      - uses: "bufbuild/buf-setup-action@v1.32.2"
         with:
           version: "1.30.0"
       - name: "push release name to BSR"

authzed/api/materialize/v0/watchpermissions.proto

@@ -0,0 +1,86 @@
+syntax = "proto3";
+package authzed.api.materialize.v0;
+
+import "authzed/api/v1/core.proto";
+
+option go_package = "github.com/authzed/authzed-go/proto/authzed/api/materialize/v0";
+option java_package = "com.authzed.api.materialize.v0";
+option java_multiple_files = true;
+
+service WatchPermissionsService {
+  // WatchPermissions returns a stream of PermissionChange events for the given permissions.
+  //
+  // WatchPermissions is a long-running RPC, and will stream events until the client
+  // closes the connection or the server terminates the stream. The consumer is responsible of
+  // keeping track of the last seen revision and resuming the stream from that point in the event
+  // of disconnection or client-side restarts.
+  //
+  // The API does not offer a sharding mechanism and thus there should only be one consumer per target system.
+  // Implementing an active-active HA consumer setup over the same target system will require coordinating which
+  // revisions have been consumed in order to prevent transitioning to an inconsistent state.
+  //
+  // Usage of WatchPermissions requires to be explicitly enabled on the service, including the permissions to be
+  // watched. It requires more resources and is less performant than WatchPermissionsSets. It's usage
+  // is only recommended when performing the set intersections of WatchPermissionSets in the client side is not viable
+  // or there is a strict application requirement to use consume the computed permissions.
+  rpc WatchPermissions(WatchPermissionsRequest) returns (stream WatchPermissionsResponse) {}
+}
+
+message WatchPermissionsRequest {
+  // permissions is a list of permissions to watch for changes. At least one permission must be specified, and it must
+  // be a subset or equal to the permissions that were enabled for the service.
+  repeated WatchedPermission permissions = 1;
+  // optional_starting_after is the revision token to start watching from. If not provided, the stream
+  // will start from the current revision at the moment of the request.
+  authzed.api.v1.ZedToken optional_starting_after = 2;
+}
+
+message WatchedPermission {
+  // resource_type is the type of the resource to watch for changes.
+  string resource_type = 1;
+  // permission is the permission to watch for changes.
+  string permission = 2;
+  // subject_type is the type of the subject to watch for changes.
+  string subject_type = 3;
+  // optional_subject_relation is the relation on the subject to watch for changes.
+  string optional_subject_relation = 4;
+}
+
+message WatchPermissionsResponse {
+  oneof response {
+    // change is the computed permission delta that has occurred as result of a mutation in origin SpiceDB.
+    // The consumer should apply this change to the current state of the computed permissions in their target system.
+    // Once an event arrives with completed_revision instead, the consumer shall consider there are not more changes
+    // originating from that revision.
+    //
+    // The consumer should keep track of the revision in order to resume streaming in the event of consumer restarts.
+    PermissionChange change = 1;
+
+    // completed_revision is the revision token that indicates all changes originating from a revision have been
+    // streamed and thus the revision should be considered completed. It may also be
+    // received without accompanying set of changes, indicating that a mutation in the origin SpiceDB cluster did
+    // not yield any effective changes in the computed permissions
+    authzed.api.v1.ZedToken completed_revision = 2;
+  }
+}
+
+message PermissionChange {
+  enum Permissionship {
+    PERMISSIONSHIP_UNSPECIFIED = 0;
+    PERMISSIONSHIP_NO_PERMISSION = 1;
+    PERMISSIONSHIP_HAS_PERMISSION = 2;
+    PERMISSIONSHIP_CONDITIONAL_PERMISSION = 3;
+  }
+
+  // revision represents the revision at which the change occurred.
+  authzed.api.v1.ZedToken revision = 1;
+
+  // resource is the resource that the permission change is related to.
+  authzed.api.v1.ObjectReference resource = 2;
+  // permission is the permission that has changed.
+  string permission = 3;
+  // subject is the subject that the permission change is related to.
+  authzed.api.v1.SubjectReference subject = 4;
+  // permissionship is the new permissionship of the subject over the resource after the change.
+  Permissionship permissionship = 5;
+}

authzed/api/materialize/v0/watchpermissionsets.proto

@@ -0,0 +1,166 @@
+syntax = "proto3";
+package authzed.api.materialize.v0;
+
+import "authzed/api/v1/core.proto";
+
+option go_package = "github.com/authzed/authzed-go/proto/authzed/api/materialize/v0";
+option java_package = "com.authzed.api.materialize.v0";
+option java_multiple_files = true;
+
+service WatchPermissionSetsService {
+  // WatchPermissionSets returns a stream of changes to the sets which can be used to compute the watched permissions.
+  //
+  // WatchPermissionSets lets consumers achieve the same thing as WatchPermissions, but trades off a simpler usage model with
+  // significantly lower computational requirements. Unlike WatchPermissions, this method returns changes to the sets of permissions,
+  // rather than the individual permissions. Permission sets are a normalized form of the computed permissions, which
+  // means that the consumer must perform an extra computation over this representation to obtain the final computed
+  // permissions, typically by intersecting the provided sets.
+  //
+  // For example, this would look like a JOIN between the
+  // materialize permission sets table in a target relation database, the table with the resources to authorize access
+  // to, and the table with the subject (e.g. a user).
+  //
+  // In exchange, the number of changes issued by WatchPermissionSets will be several orders of magnitude less than those
+  // emitted by WatchPermissions, which has several implications:
+  // - significantly less resources to compute the sets
+  // - significantly less messages to stream over the network
+  // - significantly less events to ingest on the consumer side
+  // - less ingestion lag from the origin SpiceDB mutation
+  //
+  // The type of scenarios WatchPermissionSets is particularly well suited is when a single change
+  // in the origin SpiceDB can yield millions of changes. For example, in the GitHub authorization model, assigning a role
+  // to a top-level team of an organization with hundreds of thousands of employees can lead to an explosion of
+  // permission change events that would require a lot of computational resources to process, both on Materialize and
+  // the consumer side.
+  //
+  // WatchPermissionSets is thus recommended for any larger scale use case where the fan-out in permission changes that
+  // emerges from a specific schema and data shape is too large to handle effectively.
+  //
+  // The API does not offer a sharding mechanism and thus there should only be one consumer per target system.
+  // Implementing an active-active HA consumer setup over the same target system will require coordinating which
+  // revisions have been consumed in order to prevent transitioning to an inconsistent state.
+  rpc WatchPermissionSets(WatchPermissionSetsRequest) returns (stream WatchPermissionSetsResponse) {}
+
+  // LookupPermissionSets returns the current state of the permission sets which can be used to derive the computed permissions.
+  // It's typically used to backfill the state of the permission sets in the consumer side.
+  //
+  // It's a cursored API and the consumer is responsible to keep track of the cursor and use it on each subsequent call.
+  // Each stream will return <N> permission sets defined by the specified request limit. The server will keep streaming until
+  // the sets per stream is hit, or the current state of the sets is reached,
+  // whatever happens first, and then close the stream. The server will indicate there are no more changes to stream
+  // through the `completed_members` in the cursor.
+  //
+  // There may be many elements to stream, and so the consumer should be prepared to resume the stream from the last
+  // cursor received. Once completed, the consumer may start streaming permission set changes using WatchPermissionSets
+  // and the revision token from the last LookupPermissionSets response.
+  rpc LookupPermissionSets(LookupPermissionSetsRequest) returns (stream LookupPermissionSetsResponse) {}
+}
+
+message WatchPermissionSetsRequest {
+  // optional_starting_after is used to specify the SpiceDB revision to start watching from.
+  // If not specified, the watch will start from the current SpiceDB revision time of the request ("head revision").
+  authzed.api.v1.ZedToken optional_starting_after = 1;
+}
+
+message WatchPermissionSetsResponse {
+  oneof response {
+    // change is the permission set delta that has occurred as result of a mutation in origin SpiceDB.
+    // The consumer should apply this change to the current state of the permission sets in their target system.
+    // Once an event arrives with completed_revision instead, the consumer shall consider the set of
+    // changes originating from that revision completed.
+    //
+    // The consumer should keep track of the revision in order to resume streaming in the event of consumer restarts.
+    PermissionSetChange change = 1;
+
+    // completed_revision is the revision token that indicates the completion of a set of changes. It may also be
+    // received without accompanying set of changes, indicating that a mutation in the origin SpiceDB cluster did
+    // not yield any effective changes in the permission sets
+    authzed.api.v1.ZedToken completed_revision = 2;
+
+    // lookup_permission_sets_required is a signal that the consumer should perform a LookupPermissionSets call because
+    // the permission set snapshot needs to be rebuilt from scratch. This typically happens when the origin SpiceDB
+    // cluster has seen its schema changed.
+    LookupPermissionSetsRequired lookup_permission_sets_required = 3;
+  }
+}
+
+message Cursor {
+  // limit is the number of permission sets to stream over a single LookupPermissionSets call that was requested.
+  uint32 limit = 1;
+  // token is the snapshot revision at which the cursor was computed.
+  authzed.api.v1.ZedToken token = 4;
+  // starting_index is an offset of the permission set represented by this cursor
+  uint32 starting_index = 5;
+  // completed_members is a boolean flag that indicates that the cursor has reached the end of the permission sets
+  bool completed_members = 6;
+}
+
+message LookupPermissionSetsRequest {
+  // limit is the number of permission sets to stream over a single LookupPermissionSets. Once the limit is reached,
+  // the server will close the stream. If more permission sets are available, the consume should open a new stream
+  // providing optional_starting_after_cursor, using the cursor from the last response.
+  uint32 limit = 1;
+  // optional_starting_after_cursor is used to specify the offset to start streaming permission sets from.
+  Cursor optional_starting_after_cursor = 4;
+}
+
+message LookupPermissionSetsResponse {
+  // change represents the permission set delta necessary to transition an uninitialized target system to
+  // a specific snapshot revision. In practice it's not different from the WatchPermissionSetsResponse.change, except
+  // all changes will be of time SET_OPERATION_ADDED because it's assumed there is no known previous state.
+  //
+  // Applying the deltas to a previously initialized target system would yield incorrect results.
+  PermissionSetChange change = 1;
+  // cursor points to a specific permission set in a revision.
+  // The consumer should keep track of the cursor in order to resume streaming in the event of consumer restarts. This
+  // is particularly important in backfill scenarios that may take hours or event days to complete.
+  Cursor cursor = 2;
+}
+
+message PermissionSetChange {
+  enum SetOperation {
+    SET_OPERATION_UNSPECIFIED = 0;
+    SET_OPERATION_ADDED = 1;
+    SET_OPERATION_REMOVED = 2;
+  }
+
+  // revision represents the revision at which the permission set change occurred.
+  authzed.api.v1.ZedToken at_revision = 1;
+  // operation represents the type of set operation that took place as part of the change
+  SetOperation operation = 2;
+  // parent_set represents the permission set parent of either another set or a member
+  SetReference parent_set = 3;
+
+  oneof child {
+    // child_set represents the scenario where another set is considered member of the parent set
+    SetReference child_set = 4;
+    // child_member represents the scenario where an specific object is considered member of the parent set
+    MemberReference child_member = 5;
+  }
+}
+
+message SetReference {
+    // object_type is the type of object in a permission set
+    string object_type = 1;
+    // object_id is the ID of a permission set
+    string object_id = 2;
+    // permission_or_relation is the permission or relation referenced by this permission set
+    string permission_or_relation = 3;
+}
+
+message MemberReference {
+    // object_type is the type of object of a permission set member
+    string object_type = 1;
+    // object_id is the ID of a permission set member
+    string object_id = 2;
+    // optional_permission_or_relation is the permission or relation referenced by this permission set member
+    string optional_permission_or_relation = 3;
+}
+
+// LookupPermissionSetsRequired is a signal that the consumer should perform a LookupPermissionSets call because
+// the permission set snapshot needs to be rebuilt from scratch. This typically happens when the origin SpiceDB
+// cluster has seen its schema changed.
+message LookupPermissionSetsRequired {
+  // required_lookup_at is the snapshot revision at which the permission set needs to be rebuilt to.
+  authzed.api.v1.ZedToken required_lookup_at = 1;
+}

buf.yaml

@@ -6,6 +6,8 @@ deps:
   - "buf.build/googleapis/googleapis"
   - "buf.build/grpc-ecosystem/grpc-gateway"
 lint:
+  except:
+    - "PACKAGE_VERSION_SUFFIX"
   ignore:
     - "authzed/api/v0"  # legacy from before we used buf
 breaking: