WIP: Add bazel info starlark-environments-proto

src/main/java/com/google/devtools/build/docgen/BUILD

@@ -15,12 +15,49 @@ filegroup(
     visibility = ["//src:__subpackages__"],
 )
 
+# TODO: Move those to a separate package
+java_library(
+    name = "visible_to_info",
+    srcs = [
+        "DocLinkMap.java",
+        "DocgenConsts.java",
+        "RuleLinkExpander.java",
+    ],
+    resources = [":bazel_link_map"],
+    visibility = ["//src/main/java/com/google/devtools/build/lib/runtime/commands/info:__pkg__"],
+    # TODO: Just copied all from docgen_javalib.
+    deps = [
+        "//src/main/java/com/google/devtools/build/docgen/annot",
+        "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
+        "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
+        "//src/main/java/com/google/devtools/build/lib/concurrent",
+        "//src/main/java/com/google/devtools/build/lib/packages",
+        "//src/main/java/com/google/devtools/build/lib/starlarkdocextract:labelrenderer",
+        "//src/main/java/com/google/devtools/build/lib/util:classpath",
+        "//src/main/java/com/google/devtools/build/lib/util:filetype",
+        "//src/main/java/com/google/devtools/common/options",
+        "//src/main/java/net/starlark/java/annot",
+        "//src/main/java/net/starlark/java/eval",
+        "//src/main/protobuf:builtin_java_proto",
+        "//src/main/protobuf:stardoc_output_java_proto",
+        "//third_party:apache_velocity",
+        "//third_party:auto_value",
+        "//third_party:flogger",
+        "//third_party:gson",
+        "//third_party:guava",
+        "//third_party:jsr305",
+        "@com_google_protobuf//:protobuf_java",
+    ],
+)
+
 java_library(
     name = "docgen_javalib",
     srcs = glob(["**/*.java"]),
     resources = [":template_files"],
     runtime_deps = [
-        "//src/main/java/com/google/devtools/build/lib/bazel:main",
+        # TODO: Commented out for now to avoid cyclic deps for rule_src_doc_exporter.
+        #        "//src/main/java/com/google/devtools/build/lib/bazel:main",
     ],
     deps = [
         "//src/main/java/com/google/devtools/build/docgen/annot",
@@ -36,6 +73,7 @@ java_library(
         "//src/main/java/net/starlark/java/annot",
         "//src/main/java/net/starlark/java/eval",
         "//src/main/protobuf:builtin_java_proto",
+        "//src/main/protobuf:rule_src_doc_java_proto",
         "//src/main/protobuf:stardoc_output_java_proto",
         "//third_party:apache_velocity",
         "//third_party:auto_value",
@@ -87,6 +125,26 @@ java_binary(
     ],
 )
 
+java_binary(
+    name = "rule_src_doc_exporter",
+    srcs = ["RuleSrcDocExporter.java"],
+    main_class = "com.google.devtools.build.docgen.RuleSrcDocExporter",
+    runtime_deps = [
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository",
+        "//src/main/java/com/google/devtools/build/lib/bazel/rules",
+        "//src/main/java/net/starlark/java/syntax",
+    ],
+    deps = [
+        ":docgen_javalib",
+        "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
+        "//src/main/java/com/google/devtools/common/options",
+        "//src/main/java/net/starlark/java/annot",
+        "//src/main/java/net/starlark/java/eval",
+        "//src/main/protobuf:rule_src_doc_java_proto",
+        "//third_party:guava",
+    ],
+)
+
 filegroup(
     name = "template_files",
     srcs = glob([

src/main/java/com/google/devtools/build/docgen/DocLinkMap.java

@@ -48,9 +48,17 @@ public class DocLinkMap {
 
   public static DocLinkMap createFromFile(String filePath) {
     try {
-      return new Gson().fromJson(Files.readString(Paths.get(filePath)), DocLinkMap.class);
-    } catch (IOException | JsonSyntaxException ex) {
+      return createFromString(Files.readString(Paths.get(filePath)));
+    } catch (IOException ex) {
       throw new IllegalArgumentException("Failed to read link map from " + filePath, ex);
     }
   }
+
+  public static DocLinkMap createFromString(String content) {
+    try {
+      return new Gson().fromJson(content, DocLinkMap.class);
+    } catch (JsonSyntaxException ex) {
+      throw new IllegalArgumentException("Failed to parse link map", ex);
+    }
+  }
 }

src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java

@@ -280,6 +280,11 @@ public boolean isMandatory() {
     return mandatory;
   }
 
+  /** Returns string representation of default value, if any. */
+  public @Nullable String getDefaultValue() {
+    return this.defaultValue;
+  }
+
   /** Returns a string containing the synopsis for this attribute. */
   public String getSynopsis() throws BuildEncyclopediaDocException {
     if (type == null) {

src/main/java/com/google/devtools/build/docgen/RuleLinkExpander.java

@@ -55,7 +55,7 @@ public class RuleLinkExpander {
   private final Map<String, String> ruleIndex = new HashMap<>();
   private final boolean singlePage;
 
-  RuleLinkExpander(Map<String, String> ruleIndex, boolean singlePage, DocLinkMap linkMap) {
+  public RuleLinkExpander(Map<String, String> ruleIndex, boolean singlePage, DocLinkMap linkMap) {
     this.ruleIndex.putAll(ruleIndex);
     this.ruleIndex.putAll(FUNCTIONS);
     this.singlePage = singlePage;

src/main/java/com/google/devtools/build/docgen/RuleSrcDocExporter.java

@@ -0,0 +1,135 @@
+// Copyright 2025 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.docgen;
+
+import com.google.devtools.build.docgen.starlark.*;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.build.docgen.rulesrcdoc.RuleSrcDoc.RuleSrcDocs;
+import com.google.devtools.build.docgen.rulesrcdoc.RuleSrcDoc.RuleDocumentationProto;
+import com.google.devtools.build.docgen.rulesrcdoc.RuleSrcDoc.RuleDocumentationAttributeProto;
+
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.*;
+
+/** The main class for exporting rule docs collected from source files. */
+public class RuleSrcDocExporter {
+
+  private static void printUsage(OptionsParser parser) {
+//    System.err.println(
+//        "Usage: api_exporter_bin -m link_map_path -p rule_class_provider\n"
+//            + "    [-r input_root] (-i input_dir)+ (--input_stardoc_proto binproto)+\n"
+//            + "    -f outputFile [-b denylist] [-h]\n\n"
+//            + "Exports all Starlark builtins to a file including the embedded native rules.\n"
+//            + "The link map path (-m), rule class provider (-p), output file (-f), and at least\n"
+//            + " one input_dir (-i) or binproto (--input_stardoc_proto) must be specified.\n");
+//    System.err.println(
+//        parser.describeOptionsWithDeprecatedCategories(
+//            Collections.<String, String>emptyMap(), OptionsParser.HelpVerbosity.LONG));
+  }
+
+  private static void fail(Throwable e, boolean printStackTrace) {
+    System.err.println("ERROR: " + e.getMessage());
+    if (printStackTrace) {
+      e.printStackTrace();
+    }
+    Runtime.getRuntime().exit(1);
+  }
+
+  private static ConfiguredRuleClassProvider createRuleClassProvider(String classProvider)
+          throws NoSuchMethodException, InvocationTargetException, IllegalAccessException,
+          ClassNotFoundException {
+    Class<?> providerClass = Class.forName(classProvider);
+    Method createMethod = providerClass.getMethod("create");
+    return (ConfiguredRuleClassProvider) createMethod.invoke(null);
+  }
+
+  private static RuleSrcDocs ruleDocsToProto(Collection<RuleDocumentation> ruleDocs) throws BuildEncyclopediaDocException {
+    RuleSrcDocs.Builder builder = RuleSrcDocs.newBuilder();
+
+    for (RuleDocumentation ruleDoc : ruleDocs.stream().sorted().toList()) {
+        RuleDocumentationProto.Builder ruleProto = builder.addRuleBuilder();
+        ruleProto.setRuleName(ruleDoc.getRuleName());
+        ruleProto.setHtmlDocumentation(ruleDoc.getHtmlDocumentation());
+
+        for (RuleDocumentationAttribute ruleAttr : ruleDoc.getAttributes()) {
+          RuleDocumentationAttributeProto.Builder attrProto = ruleProto.addAttributeBuilder();
+          attrProto.setAttributeName(ruleAttr.getAttributeName());
+          attrProto.setHtmlDocumentation(ruleAttr.getHtmlDocumentation());
+          String defaultValue = ruleAttr.getDefaultValue();
+          if (defaultValue != null) {
+            attrProto.setDefaultValue(defaultValue);
+          }
+          attrProto.setIsMandatory(ruleAttr.isMandatory());
+          attrProto.setIsDeprecated(ruleAttr.isDeprecated());
+        }
+    }
+    return builder.build();
+  }
+
+
+  private static void writeSrcDocs(String filename, RuleSrcDocs ruleSrcDocs) throws IOException {
+    try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(filename))) {
+      ruleSrcDocs.writeTo(out);
+    }
+  }
+
+  public static void main(String[] args) {
+    OptionsParser parser =
+            OptionsParser.builder()
+                    .optionsClasses(BuildEncyclopediaOptions.class)
+                    .allowResidue(false)
+                    .build();
+    parser.parseAndExitUponError(args);
+    BuildEncyclopediaOptions options = parser.getOptions(BuildEncyclopediaOptions.class);
+
+    if (options.help) {
+      printUsage(parser);
+      Runtime.getRuntime().exit(0);
+    }
+
+    if (options.linkMapPath.isEmpty()
+            || (options.inputJavaDirs.isEmpty() && options.inputStardocProtos.isEmpty())
+            || options.provider.isEmpty()) {
+      printUsage(parser);
+      Runtime.getRuntime().exit(1);
+    }
+
+    try {
+      DocLinkMap linkMap = DocLinkMap.createFromFile(options.linkMapPath);
+      RuleLinkExpander linkExpander = new RuleLinkExpander(
+              /* singlePage */ false, linkMap);
+      SourceUrlMapper urlMapper = new SourceUrlMapper(linkMap, options.inputRoot);
+      // TODO: Describe why RuleClassProvider is needed even when getting docs from source files.
+      ConfiguredRuleClassProvider ruleClassProvider = createRuleClassProvider(options.provider);
+
+      BuildDocCollector collector = new BuildDocCollector(linkExpander, urlMapper, ruleClassProvider);
+      Map<String, RuleDocumentation> ruleDocEntries =
+              collector.collect(options.inputJavaDirs, options.inputStardocProtos, options.denylist);
+
+      RuleSrcDocs rulesProto = ruleDocsToProto(ruleDocEntries.values());
+      writeSrcDocs(options.outputFile, rulesProto);
+
+    } catch (BuildEncyclopediaDocException e) {
+      fail(e, false);
+    } catch (Throwable e) {
+      fail(e, true);
+    }
+  }
+
+}

src/main/java/com/google/devtools/build/lib/BUILD

@@ -714,3 +714,35 @@ genrule(
         "//src/main/java/com/google/devtools/build/docgen:api_exporter",
     ],
 )
+
+genrule(
+    name = "gen_rule_src_doc_proto",
+    srcs = [
+        "//src/main/java/com/google/devtools/build/docgen:bazel_link_map",
+        "//src/main/starlark/docgen:gen_be_proto_stardoc_proto",  # TODO: Neeeded?
+        "//src/main/starlark/docgen:gen_be_java_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_cpp_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_objc_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_python_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_shell_stardoc_proto",
+        ":docs_embedded_in_sources",
+    ],
+    outs = ["rule_src_doc.pb"],
+    cmd = (
+        "$(location //src/main/java/com/google/devtools/build/docgen:rule_src_doc_exporter)" +
+        " --output_file=$@" +
+        " --link_map_path=$(location //src/main/java/com/google/devtools/build/docgen:bazel_link_map) " +
+        " --provider=com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider" +
+        " --input_root=$$PWD" +
+        " --input_dir=$$PWD/src/main/java/com/google/devtools/build/lib" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_proto_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_java_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_cpp_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_objc_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_python_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_shell_stardoc_proto)"
+    ),
+    tools = [
+        "//src/main/java/com/google/devtools/build/docgen:rule_src_doc_exporter",
+    ],
+)

src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java

@@ -58,6 +58,7 @@
 import com.google.devtools.build.lib.runtime.commands.info.ReleaseInfoItem;
 import com.google.devtools.build.lib.runtime.commands.info.ServerLogInfoItem;
 import com.google.devtools.build.lib.runtime.commands.info.ServerPidInfoItem;
+import com.google.devtools.build.lib.runtime.commands.info.StarlarkEnvironmentsProtoInfoItem;
 import com.google.devtools.build.lib.runtime.commands.info.StarlarkSemanticsInfoItem;
 import com.google.devtools.build.lib.runtime.commands.info.UsedHeapSizeAfterGcInfoItem;
 import com.google.devtools.build.lib.runtime.commands.info.UsedHeapSizeInfoItem;
@@ -297,6 +298,7 @@ private static Map<String, InfoItem> getHardwiredInfoItemMap(
             new BuildLanguageInfoItem(),
             new DefaultPackagePathInfoItem(commandOptions),
             new StarlarkSemanticsInfoItem(commandOptions),
+            new StarlarkEnvironmentsProtoInfoItem(),
             new WorkerMetricsInfoItem(),
             new LocalResourcesInfoItem());
     ImmutableMap.Builder<String, InfoItem> result = new ImmutableMap.Builder<>();

src/main/java/com/google/devtools/build/lib/runtime/commands/info/BUILD

@@ -14,12 +14,18 @@ filegroup(
 java_library(
     name = "info",
     srcs = glob(["*.java"]),
+    resources = [
+        ":gen_rule_src_doc_proto",
+    ],
     deps = [
+        "//src/main/java/com/google/devtools/build/docgen:visible_to_info",
         "//src/main/java/com/google/devtools/build/lib:runtime",
         "//src/main/java/com/google/devtools/build/lib:runtime/memory_pressure",
         "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
+        "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+        "//src/main/java/com/google/devtools/build/lib/bazel/rules",
         "//src/main/java/com/google/devtools/build/lib/bugreport",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
@@ -29,6 +35,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/pkgcache",
         "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
         "//src/main/java/com/google/devtools/build/lib/skyframe:starlark_builtins_value",
+        "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
         "//src/main/java/com/google/devtools/build/lib/util:debug-logger-configurator",
         "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
@@ -40,10 +47,45 @@ java_library(
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/com/google/devtools/common/options",
+        "//src/main/java/net/starlark/java/annot",
         "//src/main/java/net/starlark/java/eval",
         "//src/main/protobuf:build_java_proto",
+        "//src/main/protobuf:builtin_java_proto",
         "//src/main/protobuf:failure_details_java_proto",
+        "//src/main/protobuf:rule_src_doc_java_proto",
         "//third_party:flogger",
         "//third_party:guava",
     ],
 )
+
+genrule(
+    name = "gen_rule_src_doc_proto",
+    srcs = [
+        "//src/main/java/com/google/devtools/build/docgen:bazel_link_map",
+        "//src/main/starlark/docgen:gen_be_proto_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_java_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_cpp_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_objc_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_python_stardoc_proto",
+        "//src/main/starlark/docgen:gen_be_shell_stardoc_proto",
+        "//src/main/java/com/google/devtools/build/lib:docs_embedded_in_sources",
+    ],
+    outs = ["rule_src_doc.pb"],
+    cmd = (
+        "$(location //src/main/java/com/google/devtools/build/docgen:rule_src_doc_exporter)" +
+        " --output_file=$@" +
+        " --link_map_path=$(location //src/main/java/com/google/devtools/build/docgen:bazel_link_map) " +
+        " --provider=com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider" +
+        " --input_root=$$PWD" +
+        " --input_dir=$$PWD/src/main/java/com/google/devtools/build/lib" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_proto_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_java_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_cpp_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_objc_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_python_stardoc_proto)" +
+        " --input_stardoc_proto=$(location //src/main/starlark/docgen:gen_be_shell_stardoc_proto)"
+    ),
+    tools = [
+        "//src/main/java/com/google/devtools/build/docgen:rule_src_doc_exporter",
+    ],
+)