gazelle_binary: create gazelle binaries with custom languages (bazel-…

…contrib#365)

gazelle_binary is a new rule that creates binaries to be used with the
gazelle rule. It accepts a list of languages (the default languages
are proto and go). It generates a .go source file that imports the
language packages and initializes each extension. That source file is
compiled into the main package in a new gazelle binary. The rest of
the main package does not reference any specific languages.

Fixes bazel-contrib#311

README.rst

@@ -12,6 +12,9 @@ Gazelle build file generator
 .. _update: #fix-and-update
 .. _Avoiding conflicts with proto rules: https://github.com/bazelbuild/rules_go/blob/master/proto/core.rst#avoiding-conflicts
 .. _gazelle rule: #bazel-rule
+.. _Extending Gazelle: extend.rst
+.. _extended: `Extending Gazelle`_
+.. _gazelle_binary: extend.rst#gazelle_binary
 
 .. role:: cmd(code)
 .. role:: flag(code)
@@ -27,7 +30,8 @@ files for a project that follows "go build" conventions, and it can update
 existing build files to include new sources, dependencies, and options. Gazelle
 may be run by Bazel using the `gazelle rule`_, or it can be run as a command
 line tool. Gazelle can also be run in an external repository as part of the
-`go_repository`_ rule.
+`go_repository`_ rule. Gazelle may be extended_ to support new languages
+and custom rule sets.
 
 *Gazelle is under active development. Its interface and the rules it generates
 may change. Gazelle is not an official Google product.*
@@ -44,6 +48,7 @@ may change. Gazelle is not an official Google product.*
   * `git_repository`_ (deprecated)
   * `http_archive`_ (deprecated)
 
+* `Extending Gazelle`_
 * `Avoiding conflicts with proto rules`_
 
 Setup
@@ -198,8 +203,8 @@ The following attributes are available on the ``gazelle`` rule.
 +======================+=====================+======================================+
 | :param:`gazelle`     | :type:`label`       | :value:`@bazel_gazelle//cmd/gazelle` |
 +----------------------+---------------------+--------------------------------------+
-| The ``go_binary`` rule that builds Gazelle. You can substitute a modified         |
-| version of Gazelle with this.                                                     |
+| The `gazelle_binary`_ rule that builds Gazelle. You can substitute a modified     |
+| version of Gazelle with this. See `Extending Gazelle`_.                           |
 +----------------------+---------------------+--------------------------------------+
 | :param:`external`    | :type:`string`      | :value:`external`                    |
 +----------------------+---------------------+--------------------------------------+

cmd/gazelle/BUILD.bazel

@@ -1,19 +1,42 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@bazel_gazelle//:def.bzl", "DEFAULT_LANGUAGES", "gazelle_binary")
+
+gazelle_binary(
+    name = "gazelle",
+    languages = DEFAULT_LANGUAGES,
+    msan = "off",
+    pure = "off",
+    race = "off",
+    static = "off",
+    visibility = ["//visibility:public"],
+)
+
+gazelle_binary(
+    name = "gazelle_pure",
+    languages = DEFAULT_LANGUAGES,
+    msan = "off",
+    pure = "on",
+    race = "off",
+    static = "off",
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
+)
 
 go_library(
     name = "go_default_library",
+    # keep
     srcs = [
         "diff.go",
         "fix.go",
         "fix-update.go",
         "gazelle.go",
-        "langs.go",
         "print.go",
         "update-repos.go",
         "version.go",
     ],
     importpath = "github.com/bazelbuild/bazel-gazelle/cmd/gazelle",
-    visibility = ["//visibility:private"],
+    tags = ["manual"],
+    visibility = ["//visibility:public"],
     deps = [
         "//config:go_default_library",
         "//flag:go_default_library",
@@ -31,34 +54,14 @@ go_library(
     ],
 )
 
-go_binary(
-    name = "gazelle",
-    embed = [":go_default_library"],
-    msan = "off",
-    pure = "off",
-    race = "off",
-    static = "off",
-    visibility = ["//visibility:public"],
-)
-
-go_binary(
-    name = "gazelle_pure",
-    embed = [":go_default_library"],
-    msan = "off",
-    pure = "on",
-    race = "off",
-    static = "off",
-    tags = ["manual"],
-    visibility = ["//visibility:public"],
-)
-
 go_test(
     name = "go_default_test",
     size = "small",
     srcs = [
         "diff_test.go",
         "fix_test.go",
         "integration_test.go",
+        "langs.go",  # keep
     ],
     embed = [":go_default_library"],
     deps = [

cmd/gazelle/langs.go

@@ -22,6 +22,6 @@ import (
 )
 
 var languages = []language.Language{
-	proto.New(),
-	golang.New(),
+	proto.NewLanguage(),
+	golang.NewLanguage(),
 }

def.bzl

@@ -17,27 +17,43 @@ load(
     _go_context = "go_context",
     _go_rule = "go_rule",
 )
-load("@bazel_skylib//lib:shell.bzl", "shell")
-load("//internal:go_repository.bzl", _go_repository = "go_repository")
+load(
+    "@bazel_skylib//lib:shell.bzl",
+    "shell",
+)
+load(
+    "//internal:go_repository.bzl",
+    _go_repository = "go_repository",
+)
 load(
     "//internal:overlay_repository.bzl",
     _git_repository = "git_repository",
     _http_archive = "http_archive",
 )
+load(
+    "//internal:gazelle_binary.bzl",
+    _gazelle_binary = "gazelle_binary",
+)
 
 go_repository = _go_repository
 git_repository = _git_repository
 http_archive = _http_archive
+gazelle_binary = _gazelle_binary
+
+DEFAULT_LANGUAGES = [
+    "@bazel_gazelle//language/proto:go_default_library",
+    "@bazel_gazelle//language/go:go_default_library",
+]
 
 def _gazelle_runner_impl(ctx):
     go = _go_context(ctx)
     args = [
         ctx.attr.command,
         "-mode",
         ctx.attr.mode,
-        "-external",
-        ctx.attr.external,
     ]
+    if ctx.attr.external:
+        args.extend(["-external", ctx.attr.external])
     if ctx.attr.prefix:
         args.extend(["-go_prefix", ctx.attr.prefix])
     if ctx.attr.build_tags:
@@ -89,8 +105,8 @@ _gazelle_runner = _go_rule(
             default = "fix",
         ),
         "external": attr.string(
-            values = ["external", "vendored"],
-            default = "external",
+            values = ["", "external", "vendored"],
+            default = "",
         ),
         "build_tags": attr.string_list(),
         "prefix": attr.string(),

extend.rst

@@ -0,0 +1,163 @@
+Extending Gazelle
+=================
+
+.. Begin directives
+.. _Language: https://godoc.org/github.com/bazelbuild/bazel-gazelle/language#Language
+.. _`//internal/gazellebinarytest:go_default_library`: https://github.com/bazelbuild/bazel-gazelle/tree/master/internal/gazellebinarytest
+.. _`//language/go:go_default_library`: https://github.com/bazelbuild/bazel-gazelle/tree/master/language/go
+.. _`//language/proto:go_default_library`: https://github.com/bazelbuild/bazel-gazelle/tree/master/language/proto
+.. _gazelle: https://github.com/bazelbuild/bazel-gazelle#bazel-rule
+.. _go_binary: https://github.com/bazelbuild/rules_go/blob/master/go/core.rst#go-binary
+.. _go_library: https://github.com/bazelbuild/rules_go/blob/master/go/core.rst#go-library
+.. _proto godoc: https://godoc.org/github.com/bazelbuild/bazel-gazelle/language/proto
+.. _proto.GetProtoConfig: https://godoc.org/github.com/bazelbuild/bazel-gazelle/language/proto#GetProtoConfig
+.. _proto.Package: https://godoc.org/github.com/bazelbuild/bazel-gazelle/language/proto#Package
+
+.. role:: cmd(code)
+.. role:: flag(code)
+.. role:: direc(code)
+.. role:: param(kbd)
+.. role:: type(emphasis)
+.. role:: value(code)
+.. |mandatory| replace:: **mandatory value**
+.. End directives
+
+Gazelle started out as a build file generator for Go projects, but it can be
+extended to support other languages and custom sets of rules.
+
+To extend Gazelle, you must do three things:
+
+* Write a `go_library`_ with a function named ``NewLanguage`` that provides an
+  implementation of the Language_ interface. This interface provides hooks for
+  generating rules, parsing configuration directives, and resolving imports
+  to Bazel labels.
+* Write a `gazelle_binary`_ rule. Include your library in the ``languages``
+  list.
+* Write a `gazelle`_ rule that points to your ``gazelle_binary``. When you run
+  ``bazel run //:gazelle``, your binary will be built and executed instead of
+  the default binary.
+
+Example
+-------
+
+**TODO:** Add a self-contained, concise, realistic example.
+
+Gazelle itself is built using the model described above, so it may serve as
+an example.
+
+`//language/proto:go_default_library`_ and `//language/go:go_default_library`_
+both implement the `Language`_
+interface. There is also `//internal/gazellebinarytest:go_default_library`_,
+a stub implementation used for testing.
+
+``//cmd/gazelle`` is a ``gazelle_binary`` rule that includes both of these
+libraries through the ``DEFAULT_LANGUAGES`` list (you may want to use
+``DEFAULT_LANGUAGES`` in your own rule). The ``msan``, ``pure``, ``race``,
+and ``static`` attributes are optional.
+
+.. code:: bzl
+
+    load("@bazel_gazelle//:def.bzl", "DEFAULT_LANGUAGES", "gazelle_binary")
+
+    gazelle_binary(
+        name = "gazelle",
+        languages = DEFAULT_LANGUAGES,
+        msan = "off",
+        pure = "off",
+        race = "off",
+        static = "off",
+        visibility = ["//visibility:public"],
+    )
+
+This binary can be invoked using a ``gazelle`` rule like this:
+
+.. code:: bzl
+
+    load("@bazel_gazelle//:def.bzl", "gazelle")
+
+    # gazelle:prefix example.com/project
+    gazelle(
+        name = "gazelle",
+        gazelle = "//:my_gazelle_binary",
+    )
+
+You can run this with ``bazel run //:gazelle``.
+
+gazelle_binary
+--------------
+
+The ``gazelle_binary`` rule builds a Go binary that incorporates a list of
+language extensions. This requires generating a small amount of code that
+must be compiled into Gazelle's main package, so the normal `go_binary`_
+rule is not used.
+
+When the binary runs, each language extension is run sequentially. This affects
+the order that rules appear in generated build files. Metadata may be produced
+by an earlier extension and consumed by a later extension. For example, the
+proto extension stores metadata in hidden attributes of generated
+``proto_library`` rules. The Go extension uses this metadata to generate
+``go_proto_library`` rules.
+
+The following attributes are supported on the ``gazelle_binary`` rule.
+
++----------------------+---------------------+--------------------------------------+
+| **Name**             | **Type**            | **Default value**                    |
++======================+=====================+======================================+
+| :param:`languages`   | :type:`label_list`  | |mandatory|                          |
++----------------------+---------------------+--------------------------------------+
+| A list of language extensions the Gazelle binary will use.                        |
+|                                                                                   |
+| Each extension must be a `go_library`_ or something compatible. Each extension    |
+| must export a function named ``NewLanguage`` with no parameters that returns      |
+| a value assignable to `Language`_.                                                |
++----------------------+---------------------+--------------------------------------+
+| :param:`pure`        | :type:`string`      | :value:`auto`                        |
++----------------------+---------------------+--------------------------------------+
+| Same meaning as `go_binary`_. It may be necessary to set this to avoid            |
+| command flags that affect both host and target configurations.                    |
++----------------------+---------------------+--------------------------------------+
+| :param:`static`        | :type:`string`      | :value:`auto`                      |
++----------------------+---------------------+--------------------------------------+
+| Same meaning as `go_binary`_. It may be necessary to set this to avoid            |
+| command flags that affect both host and target configurations.                    |
++----------------------+---------------------+--------------------------------------+
+| :param:`race`        | :type:`string`      | :value:`auto`                        |
++----------------------+---------------------+--------------------------------------+
+| Same meaning as `go_binary`_. It may be necessary to set this to avoid            |
+| command flags that affect both host and target configurations.                    |
++----------------------+---------------------+--------------------------------------+
+| :param:`msan`        | :type:`string`      | :value:`auto`                        |
++----------------------+---------------------+--------------------------------------+
+| Same meaning as `go_binary`_. It may be necessary to set this to avoid            |
+| command flags that affect both host and target configurations.                    |
++----------------------+---------------------+--------------------------------------+
+| :param:`goos`        | :type:`string`      | :value:`auto`                        |
++----------------------+---------------------+--------------------------------------+
+| Same meaning as `go_binary`_. It may be necessary to set this to avoid            |
+| command flags that affect both host and target configurations.                    |
++----------------------+---------------------+--------------------------------------+
+| :param:`goarch`        | :type:`string`      | :value:`auto`                      |
++----------------------+---------------------+--------------------------------------+
+| Same meaning as `go_binary`_. It may be necessary to set this to avoid            |
+| command flags that affect both host and target configurations.                    |
++----------------------+---------------------+--------------------------------------+
+
+Interacting with protos
+-----------------------
+
+The proto extension (`//language/proto:go_default_library`_) gathers metadata
+from .proto files and generates ``proto_library`` rules based on that metadata.
+Extensions that generate language-specific proto rules (e.g.,
+``go_proto_library``) may use this metadata.
+
+For API reference, see the `proto godoc`_.
+
+To get proto configuration information, call `proto.GetProtoConfig`_. This is
+mainly useful for discovering the current proto mode.
+
+To get information about ``proto_library`` rules, examine the ``OtherGen``
+list of rules passed to ``language.GenerateRules``. This is a list of rules
+generated by other language extensions, and it will include ``proto_library``
+rules in each directory, if there were any. For each of these rules, you can
+call ``r.PrivateAttr(proto.PackageKey)`` to get a `proto.Package`_ record. This
+includes the proto package name, as well as source names, imports, and options.