feat: new custom linters system (#4437)

pkg/commands/custom.go

@@ -0,0 +1,81 @@
+package commands
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/golangci/golangci-lint/pkg/commands/internal"
+	"github.com/golangci/golangci-lint/pkg/logutils"
+)
+
+const envKeepTempFiles = "CUSTOM_GCL_KEEP_TEMP_FILES"
+
+type customCommand struct {
+	cmd *cobra.Command
+
+	cfg *internal.Configuration
+
+	log logutils.Log
+}
+
+func newCustomCommand(logger logutils.Log) *customCommand {
+	c := &customCommand{log: logger}
+
+	customCmd := &cobra.Command{
+		Use:     "custom",
+		Short:   "Build a version of golangci-lint with custom linters.",
+		Args:    cobra.NoArgs,
+		PreRunE: c.preRunE,
+		RunE:    c.runE,
+	}
+
+	c.cmd = customCmd
+
+	return c
+}
+
+func (c *customCommand) preRunE(_ *cobra.Command, _ []string) error {
+	cfg, err := internal.LoadConfiguration()
+	if err != nil {
+		return err
+	}
+
+	err = cfg.Validate()
+	if err != nil {
+		return err
+	}
+
+	c.cfg = cfg
+
+	return nil
+}
+
+func (c *customCommand) runE(_ *cobra.Command, _ []string) error {
+	ctx := context.Background()
+
+	tmp, err := os.MkdirTemp(os.TempDir(), "custom-gcl")
+	if err != nil {
+		return fmt.Errorf("create temporary directory: %w", err)
+	}
+
+	defer func() {
+		if os.Getenv(envKeepTempFiles) != "" {
+			log.Printf("WARN: The env var %s has been dectected: the temporary directory is preserved: %s", envKeepTempFiles, tmp)
+
+			return
+		}
+
+		_ = os.RemoveAll(tmp)
+	}()
+
+	err = internal.NewBuilder(c.log, c.cfg, tmp).Build(ctx)
+	if err != nil {
+		return fmt.Errorf("build process: %w", err)
+	}
+
+	return nil
+}

pkg/commands/help.go

@@ -53,8 +53,7 @@ func newHelpCommand(logger logutils.Log) *helpCommand {
 func (c *helpCommand) preRunE(_ *cobra.Command, _ []string) error {
 	// The command doesn't depend on the real configuration.
 	// It just needs the list of all plugins and all presets.
-	dbManager, err := lintersdb.NewManager(c.log.Child(logutils.DebugKeyLintersDB), config.NewDefault(),
-		lintersdb.NewPluginBuilder(c.log), lintersdb.NewLinterBuilder())
+	dbManager, err := lintersdb.NewManager(c.log.Child(logutils.DebugKeyLintersDB), config.NewDefault(), lintersdb.NewLinterBuilder())
 	if err != nil {
 		return err
 	}

pkg/commands/internal/builder.go

@@ -0,0 +1,219 @@
+package internal
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/golangci/golangci-lint/pkg/logutils"
+)
+
+// Builder runs all the required commands to build a binary.
+type Builder struct {
+	cfg *Configuration
+
+	log logutils.Log
+
+	root string
+	repo string
+}
+
+// NewBuilder creates a new Builder.
+func NewBuilder(logger logutils.Log, cfg *Configuration, root string) *Builder {
+	return &Builder{
+		cfg:  cfg,
+		log:  logger,
+		root: root,
+		repo: filepath.Join(root, "golangci-lint"),
+	}
+}
+
+// Build builds the custom binary.
+func (b Builder) Build(ctx context.Context) error {
+	b.log.Infof("Cloning golangci-lint repository.")
+
+	err := b.clone(ctx)
+	if err != nil {
+		return fmt.Errorf("clone golangci-lint: %w", err)
+	}
+
+	b.log.Infof("Adding plugin imports.")
+
+	err = b.updatePluginsFile()
+	if err != nil {
+		return fmt.Errorf("update plugin file: %w", err)
+	}
+
+	b.log.Infof("Adding replace directives.")
+
+	err = b.addReplaceDirectives(ctx)
+	if err != nil {
+		return fmt.Errorf("add replace directives: %w", err)
+	}
+
+	b.log.Infof("Running go mod tidy.")
+
+	err = b.goModTidy(ctx)
+	if err != nil {
+		return fmt.Errorf("go mod tidy: %w", err)
+	}
+
+	b.log.Infof("Building golangci-lint binary.")
+
+	binaryName := b.getBinaryName()
+
+	err = b.goBuild(ctx, binaryName)
+	if err != nil {
+		return fmt.Errorf("build golangci-lint binary: %w", err)
+	}
+
+	b.log.Infof("Moving golangci-lint binary.")
+
+	err = b.copyBinary(binaryName)
+	if err != nil {
+		return fmt.Errorf("move golangci-lint binary: %w", err)
+	}
+
+	return nil
+}
+
+func (b Builder) clone(ctx context.Context) error {
+	//nolint:gosec // the variable is sanitized.
+	cmd := exec.CommandContext(ctx,
+		"git", "clone", "--branch", sanitizeVersion(b.cfg.Version),
+		"--single-branch", "--depth", "1", "-c advice.detachedHead=false", "-q",
+		"https://github.com/golangci/golangci-lint.git",
+	)
+	cmd.Dir = b.root
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		b.log.Infof(string(output))
+
+		return fmt.Errorf("%s: %w", strings.Join(cmd.Args, " "), err)
+	}
+
+	return nil
+}
+
+func (b Builder) addReplaceDirectives(ctx context.Context) error {
+	for _, plugin := range b.cfg.Plugins {
+		if plugin.Path == "" {
+			continue
+		}
+
+		replace := fmt.Sprintf("%s=%s", plugin.Module, plugin.Path)
+
+		cmd := exec.CommandContext(ctx, "go", "mod", "edit", "-replace", replace)
+		cmd.Dir = b.repo
+
+		b.log.Infof("run: %s", strings.Join(cmd.Args, " "))
+
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			b.log.Warnf(string(output))
+
+			return fmt.Errorf("%s: %w", strings.Join(cmd.Args, " "), err)
+		}
+	}
+
+	return nil
+}
+
+func (b Builder) goModTidy(ctx context.Context) error {
+	cmd := exec.CommandContext(ctx, "go", "mod", "tidy")
+	cmd.Dir = b.repo
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		b.log.Warnf(string(output))
+
+		return fmt.Errorf("%s: %w", strings.Join(cmd.Args, " "), err)
+	}
+
+	return nil
+}
+
+func (b Builder) goBuild(ctx context.Context, binaryName string) error {
+	//nolint:gosec // the variable is sanitized.
+	cmd := exec.CommandContext(ctx, "go", "build",
+		"-ldflags",
+		fmt.Sprintf(
+			"-s -w -X 'main.version=%s-custom-gcl' -X 'main.date=%s'",
+			sanitizeVersion(b.cfg.Version), time.Now().UTC().String(),
+		),
+		"-o", binaryName,
+		"./cmd/golangci-lint",
+	)
+	cmd.Dir = b.repo
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		b.log.Warnf(string(output))
+
+		return fmt.Errorf("%s: %w", strings.Join(cmd.Args, " "), err)
+	}
+
+	return nil
+}
+
+func (b Builder) copyBinary(binaryName string) error {
+	src := filepath.Join(b.repo, binaryName)
+
+	source, err := os.Open(filepath.Clean(src))
+	if err != nil {
+		return fmt.Errorf("open source file: %w", err)
+	}
+
+	defer func() { _ = source.Close() }()
+
+	info, err := source.Stat()
+	if err != nil {
+		return fmt.Errorf("stat source file: %w", err)
+	}
+
+	if b.cfg.Destination != "" {
+		err = os.MkdirAll(b.cfg.Destination, os.ModePerm)
+		if err != nil {
+			return fmt.Errorf("create destination directory: %w", err)
+		}
+	}
+
+	dst, err := os.OpenFile(filepath.Join(b.cfg.Destination, binaryName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode())
+	if err != nil {
+		return fmt.Errorf("create destination file: %w", err)
+	}
+
+	defer func() { _ = dst.Close() }()
+
+	_, err = io.Copy(dst, source)
+	if err != nil {
+		return fmt.Errorf("copy source to destination: %w", err)
+	}
+
+	return nil
+}
+
+func (b Builder) getBinaryName() string {
+	name := b.cfg.Name
+	if runtime.GOOS == "windows" {
+		name += ".exe"
+	}
+
+	return name
+}
+
+func sanitizeVersion(v string) string {
+	fn := func(c rune) bool {
+		return !(unicode.IsLetter(c) || unicode.IsNumber(c) || c == '.' || c == '/')
+	}
+
+	return strings.Join(strings.FieldsFunc(v, fn), "")
+}

pkg/commands/internal/builder_test.go

@@ -0,0 +1,57 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_sanitizeVersion(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		input    string
+		expected string
+	}{
+		{
+			desc:     "ampersand",
+			input:    " te&st",
+			expected: "test",
+		},
+		{
+			desc:     "pipe",
+			input:    " te|st",
+			expected: "test",
+		},
+		{
+			desc:     "version",
+			input:    "v1.2.3",
+			expected: "v1.2.3",
+		},
+		{
+			desc:     "branch",
+			input:    "feat/test",
+			expected: "feat/test",
+		},
+		{
+			desc:     "branch",
+			input:    "value --key",
+			expected: "valuekey",
+		},
+		{
+			desc:     "hash",
+			input:    "cd8b1177",
+			expected: "cd8b1177",
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			v := sanitizeVersion(test.input)
+
+			assert.Equal(t, test.expected, v)
+		})
+	}
+}

pkg/commands/internal/configuration.go

@@ -0,0 +1,138 @@
+package internal
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+const base = ".custom-gcl"
+
+const defaultBinaryName = "custom-gcl"
+
+// Configuration represents the configuration file.
+type Configuration struct {
+	// golangci-lint version.
+	Version string `yaml:"version"`
+
+	// Name of the binary.
+	Name string `yaml:"name,omitempty"`
+
+	// Destination is the path to a directory to store the binary.
+	Destination string `yaml:"destination,omitempty"`
+
+	// Plugins information.
+	Plugins []*Plugin `yaml:"plugins,omitempty"`
+}
+
+// Validate checks and clean the configuration.
+func (c *Configuration) Validate() error {
+	if strings.TrimSpace(c.Version) == "" {
+		return errors.New("root field 'version' is required")
+	}
+
+	if strings.TrimSpace(c.Name) == "" {
+		c.Name = defaultBinaryName
+	}
+
+	if len(c.Plugins) == 0 {
+		return errors.New("no plugins defined")
+	}
+
+	for _, plugin := range c.Plugins {
+		if strings.TrimSpace(plugin.Module) == "" {
+			return errors.New("field 'module' is required")
+		}
+
+		if strings.TrimSpace(plugin.Import) == "" {
+			plugin.Import = plugin.Module
+		}
+
+		if strings.TrimSpace(plugin.Path) == "" && strings.TrimSpace(plugin.Version) == "" {
+			return errors.New("missing information: 'version' or 'path' should be provided")
+		}
+
+		if strings.TrimSpace(plugin.Path) != "" && strings.TrimSpace(plugin.Version) != "" {
+			return errors.New("invalid configuration: 'version' and 'path' should not be provided at the same time")
+		}
+
+		if strings.TrimSpace(plugin.Path) == "" {
+			continue
+		}
+
+		abs, err := filepath.Abs(plugin.Path)
+		if err != nil {
+			return err
+		}
+
+		plugin.Path = abs
+	}
+
+	return nil
+}
+
+// Plugin represents information about a plugin.
+type Plugin struct {
+	// Module name.
+	Module string `yaml:"module"`
+
+	// Import to use.
+	Import string `yaml:"import,omitempty"`
+
+	// Version of the module.
+	// Only for module available through a Go proxy.
+	Version string `yaml:"version,omitempty"`
+
+	// Path to the local module.
+	// Only for local module.
+	Path string `yaml:"path,omitempty"`
+}
+
+func LoadConfiguration() (*Configuration, error) {
+	configFilePath, err := findConfigurationFile()
+	if err != nil {
+		return nil, fmt.Errorf("file %s not found: %w", configFilePath, err)
+	}
+
+	file, err := os.Open(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("file %s open: %w", configFilePath, err)
+	}
+
+	var cfg Configuration
+
+	err = yaml.NewDecoder(file).Decode(&cfg)
+	if err != nil {
+		return nil, fmt.Errorf("YAML decoding: %w", err)
+	}
+
+	return &cfg, nil
+}
+
+func findConfigurationFile() (string, error) {
+	entries, err := os.ReadDir(".")
+	if err != nil {
+		return "", fmt.Errorf("read directory: %w", err)
+	}
+
+	for _, entry := range entries {
+		ext := filepath.Ext(entry.Name())
+
+		switch strings.ToLower(strings.TrimPrefix(ext, ".")) {
+		case "yml", "yaml", "json":
+			if isConf(ext, entry.Name()) {
+				return entry.Name(), nil
+			}
+		}
+	}
+
+	return "", errors.New("configuration file not found")
+}
+
+func isConf(ext, name string) bool {
+	return base+ext == name
+}

pkg/commands/internal/configuration_test.go

@@ -0,0 +1,127 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfiguration_Validate(t *testing.T) {
+	testCases := []struct {
+		desc string
+		cfg  *Configuration
+	}{
+		{
+			desc: "version",
+			cfg: &Configuration{
+				Version: "v1.57.0",
+				Plugins: []*Plugin{
+					{
+						Module:  "example.org/foo/bar",
+						Import:  "example.org/foo/bar/test",
+						Version: "v1.2.3",
+					},
+				},
+			},
+		},
+		{
+			desc: "path",
+			cfg: &Configuration{
+				Version: "v1.57.0",
+				Plugins: []*Plugin{
+					{
+						Module: "example.org/foo/bar",
+						Import: "example.org/foo/bar/test",
+						Path:   "/my/path",
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			err := test.cfg.Validate()
+			require.NoError(t, err)
+		})
+	}
+}
+
+func TestConfiguration_Validate_error(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		cfg      *Configuration
+		expected string
+	}{
+		{
+			desc:     "missing version",
+			cfg:      &Configuration{},
+			expected: "root field 'version' is required",
+		},
+		{
+			desc: "no plugins",
+			cfg: &Configuration{
+				Version: "v1.57.0",
+			},
+			expected: "no plugins defined",
+		},
+		{
+			desc: "missing module",
+			cfg: &Configuration{
+				Version: "v1.57.0",
+				Plugins: []*Plugin{
+					{
+						Module:  "",
+						Import:  "example.org/foo/bar/test",
+						Version: "v1.2.3",
+						Path:    "/my/path",
+					},
+				},
+			},
+			expected: "field 'module' is required",
+		},
+		{
+			desc: "module version and path",
+			cfg: &Configuration{
+				Version: "v1.57.0",
+				Plugins: []*Plugin{
+					{
+						Module:  "example.org/foo/bar",
+						Import:  "example.org/foo/bar/test",
+						Version: "v1.2.3",
+						Path:    "/my/path",
+					},
+				},
+			},
+			expected: "invalid configuration: 'version' and 'path' should not be provided at the same time",
+		},
+		{
+			desc: "no module version and path",
+			cfg: &Configuration{
+				Version: "v1.57.0",
+				Plugins: []*Plugin{
+					{
+						Module: "example.org/foo/bar",
+						Import: "example.org/foo/bar/test",
+					},
+				},
+			},
+			expected: "missing information: 'version' or 'path' should be provided",
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			t.Parallel()
+
+			err := test.cfg.Validate()
+
+			assert.EqualError(t, err, test.expected)
+		})
+	}
+}

pkg/commands/internal/imports.go

@@ -0,0 +1,69 @@
+package internal
+
+import (
+	"bytes"
+	"fmt"
+	"go/format"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+const importsTemplate = `
+package main
+
+import (
+{{range .Imports -}}
+	_ "{{.}}"
+{{end -}}
+)
+`
+
+func (b Builder) updatePluginsFile() error {
+	importsDest := filepath.Join(b.repo, "cmd", "golangci-lint", "plugins.go")
+
+	info, err := os.Stat(importsDest)
+	if err != nil {
+		return fmt.Errorf("file %s not found: %w", importsDest, err)
+	}
+
+	source, err := generateImports(b.cfg)
+	if err != nil {
+		return fmt.Errorf("generate imports: %w", err)
+	}
+
+	b.log.Infof("generated imports info %s:\n%s\n", importsDest, source)
+
+	err = os.WriteFile(filepath.Clean(importsDest), source, info.Mode())
+	if err != nil {
+		return fmt.Errorf("write file %s: %w", importsDest, err)
+	}
+
+	return nil
+}
+
+func generateImports(cfg *Configuration) ([]byte, error) {
+	impTmpl, err := template.New("plugins.go").Parse(importsTemplate)
+	if err != nil {
+		return nil, fmt.Errorf("parse template: %w", err)
+	}
+
+	var imps []string
+	for _, plugin := range cfg.Plugins {
+		imps = append(imps, plugin.Import)
+	}
+
+	buf := &bytes.Buffer{}
+
+	err = impTmpl.Execute(buf, map[string]any{"Imports": imps})
+	if err != nil {
+		return nil, fmt.Errorf("execute template: %w", err)
+	}
+
+	source, err := format.Source(buf.Bytes())
+	if err != nil {
+		return nil, fmt.Errorf("format source: %w", err)
+	}
+
+	return source, nil
+}

pkg/commands/internal/imports_test.go

@@ -0,0 +1,36 @@
+package internal
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_generateImports(t *testing.T) {
+	cfg := &Configuration{
+		Version: "v1.57.0",
+		Plugins: []*Plugin{
+			{
+				Module:  "example.org/foo/bar",
+				Import:  "example.org/foo/bar/test",
+				Version: "v1.2.3",
+			},
+			{
+				Module: "example.com/foo/bar",
+				Import: "example.com/foo/bar/test",
+				Path:   "/my/path",
+			},
+		},
+	}
+
+	data, err := generateImports(cfg)
+	require.NoError(t, err)
+
+	expected, err := os.ReadFile(filepath.Join("testdata", "imports.go"))
+	require.NoError(t, err)
+
+	assert.Equal(t, expected, data)
+}

pkg/commands/internal/testdata/imports.go

@@ -0,0 +1,6 @@
+package main
+
+import (
+	_ "example.com/foo/bar/test"
+	_ "example.org/foo/bar/test"
+)

pkg/commands/linters.go

@@ -65,7 +65,7 @@ func (c *lintersCommand) preRunE(cmd *cobra.Command, _ []string) error {
 	}
 
 	dbManager, err := lintersdb.NewManager(c.log.Child(logutils.DebugKeyLintersDB), c.cfg,
-		lintersdb.NewPluginBuilder(c.log), lintersdb.NewLinterBuilder())
+		lintersdb.NewLinterBuilder(), lintersdb.NewPluginModuleBuilder(c.log), lintersdb.NewPluginGoBuilder(c.log))
 	if err != nil {
 		return err
 	}

pkg/commands/root.go

@@ -63,6 +63,7 @@ func newRootCommand(info BuildInfo) *rootCommand {
 		newCacheCommand().cmd,
 		newConfigCommand(log).cmd,
 		newVersionCommand(info).cmd,
+		newCustomCommand(log).cmd,
 	)
 
 	rootCmd.SetHelpCommand(newHelpCommand(log).cmd)

pkg/commands/run.go

@@ -180,7 +180,7 @@ func (c *runCommand) persistentPostRunE(_ *cobra.Command, _ []string) error {
 
 func (c *runCommand) preRunE(_ *cobra.Command, _ []string) error {
 	dbManager, err := lintersdb.NewManager(c.log.Child(logutils.DebugKeyLintersDB), c.cfg,
-		lintersdb.NewPluginBuilder(c.log), lintersdb.NewLinterBuilder())
+		lintersdb.NewLinterBuilder(), lintersdb.NewPluginModuleBuilder(c.log), lintersdb.NewPluginGoBuilder(c.log))
 	if err != nil {
 		return err
 	}