From d248d0ff2f6cc0f4a110e6196ae98f2064e331d5 Mon Sep 17 00:00:00 2001 From: berk-karaal Date: Wed, 5 Mar 2025 23:53:31 +0300 Subject: [PATCH 1/3] feat(codegen/golang): allow exporting models file to separate package --- internal/codegen/golang/gen.go | 48 ++++++++++++----- internal/codegen/golang/imports.go | 53 ++++++++++++++++--- internal/codegen/golang/opts/options.go | 10 +++- internal/codegen/golang/output_file.go | 12 +++++ internal/codegen/golang/postgresql_type.go | 15 ++++-- internal/codegen/golang/query.go | 2 +- internal/codegen/golang/result.go | 3 +- internal/codegen/golang/struct.go | 8 +++ .../codegen/golang/templates/template.tmpl | 2 +- 9 files changed, 125 insertions(+), 28 deletions(-) create mode 100644 internal/codegen/golang/output_file.go diff --git a/internal/codegen/golang/gen.go b/internal/codegen/golang/gen.go index 5b7977f500..14a6252aa8 100644 --- a/internal/codegen/golang/gen.go +++ b/internal/codegen/golang/gen.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "go/format" + "path/filepath" "strings" "text/template" @@ -122,7 +123,7 @@ func Generate(ctx context.Context, req *plugin.GenerateRequest) (*plugin.Generat } if options.OmitUnusedStructs { - enums, structs = filterUnusedStructs(enums, structs, queries) + enums, structs = filterUnusedStructs(options, enums, structs, queries) } if err := validate(options, enums, structs, queries); err != nil { @@ -211,6 +212,7 @@ func generate(req *plugin.GenerateRequest, options *opts.Options, enums []Enum, "imports": i.Imports, "hasImports": i.HasImports, "hasPrefix": strings.HasPrefix, + "trimPrefix": strings.TrimPrefix, // These methods are Go specific, they do not belong in the codegen package // (as that is language independent) @@ -232,7 +234,7 @@ func generate(req *plugin.GenerateRequest, options *opts.Options, enums []Enum, output := map[string]string{} - execute := func(name, templateName string) error { + execute := func(name, packageName, templateName string) error { imports := i.Imports(name) replacedQueries := replaceConflictedArg(imports, queries) @@ -240,6 +242,7 @@ func generate(req *plugin.GenerateRequest, options *opts.Options, enums []Enum, w := bufio.NewWriter(&b) tctx.SourceName = name tctx.GoQueries = replacedQueries + tctx.Package = packageName err := tmpl.ExecuteTemplate(w, templateName, &tctx) w.Flush() if err != nil { @@ -251,8 +254,13 @@ func generate(req *plugin.GenerateRequest, options *opts.Options, enums []Enum, return fmt.Errorf("source error: %w", err) } - if templateName == "queryFile" && options.OutputFilesSuffix != "" { - name += options.OutputFilesSuffix + if templateName == "queryFile" { + if options.OutputQueryFilesDirectory != "" { + name = filepath.Join(options.OutputQueryFilesDirectory, name) + } + if options.OutputFilesSuffix != "" { + name += options.OutputFilesSuffix + } } if !strings.HasSuffix(name, ".go") { @@ -284,24 +292,29 @@ func generate(req *plugin.GenerateRequest, options *opts.Options, enums []Enum, batchFileName = options.OutputBatchFileName } - if err := execute(dbFileName, "dbFile"); err != nil { + modelsPackageName := options.Package + if options.OutputModelsPackage != "" { + modelsPackageName = options.OutputModelsPackage + } + + if err := execute(dbFileName, options.Package, "dbFile"); err != nil { return nil, err } - if err := execute(modelsFileName, "modelsFile"); err != nil { + if err := execute(modelsFileName, modelsPackageName, "modelsFile"); err != nil { return nil, err } if options.EmitInterface { - if err := execute(querierFileName, "interfaceFile"); err != nil { + if err := execute(querierFileName, options.Package, "interfaceFile"); err != nil { return nil, err } } if tctx.UsesCopyFrom { - if err := execute(copyfromFileName, "copyfromFile"); err != nil { + if err := execute(copyfromFileName, options.Package, "copyfromFile"); err != nil { return nil, err } } if tctx.UsesBatch { - if err := execute(batchFileName, "batchFile"); err != nil { + if err := execute(batchFileName, options.Package, "batchFile"); err != nil { return nil, err } } @@ -312,7 +325,7 @@ func generate(req *plugin.GenerateRequest, options *opts.Options, enums []Enum, } for source := range files { - if err := execute(source, "queryFile"); err != nil { + if err := execute(source, options.Package, "queryFile"); err != nil { return nil, err } } @@ -362,7 +375,7 @@ func checkNoTimesForMySQLCopyFrom(queries []Query) error { return nil } -func filterUnusedStructs(enums []Enum, structs []Struct, queries []Query) ([]Enum, []Struct) { +func filterUnusedStructs(options *opts.Options, enums []Enum, structs []Struct, queries []Query) ([]Enum, []Struct) { keepTypes := make(map[string]struct{}) for _, query := range queries { @@ -389,8 +402,15 @@ func filterUnusedStructs(enums []Enum, structs []Struct, queries []Query) ([]Enu keepEnums := make([]Enum, 0, len(enums)) for _, enum := range enums { - _, keep := keepTypes[enum.Name] - _, keepNull := keepTypes["Null"+enum.Name] + var enumType string + if options.ModelsPackageImportPath != "" { + enumType = options.OutputModelsPackage + "." + enum.Name + } else { + enumType = enum.Name + } + + _, keep := keepTypes[enumType] + _, keepNull := keepTypes["Null"+enumType] if keep || keepNull { keepEnums = append(keepEnums, enum) } @@ -398,7 +418,7 @@ func filterUnusedStructs(enums []Enum, structs []Struct, queries []Query) ([]Enu keepStructs := make([]Struct, 0, len(structs)) for _, st := range structs { - if _, ok := keepTypes[st.Name]; ok { + if _, ok := keepTypes[st.Type()]; ok { keepStructs = append(keepStructs, st) } } diff --git a/internal/codegen/golang/imports.go b/internal/codegen/golang/imports.go index 9e7819e4b1..2c4417cd22 100644 --- a/internal/codegen/golang/imports.go +++ b/internal/codegen/golang/imports.go @@ -160,7 +160,7 @@ var pqtypeTypes = map[string]struct{}{ "pqtype.NullRawMessage": {}, } -func buildImports(options *opts.Options, queries []Query, uses func(string) bool) (map[string]struct{}, map[ImportSpec]struct{}) { +func buildImports(options *opts.Options, queries []Query, outputFile OutputFile, uses func(string) bool) (map[string]struct{}, map[ImportSpec]struct{}) { pkg := make(map[ImportSpec]struct{}) std := make(map[string]struct{}) @@ -243,11 +243,52 @@ func buildImports(options *opts.Options, queries []Query, uses func(string) bool } } + requiresModelsPackageImport := func() bool { + if options.ModelsPackageImportPath == "" { + return false + } + + for _, q := range queries { + // Check if the return type is from models package (possibly a model struct or an enum) + if q.hasRetType() && strings.HasPrefix(q.Ret.Type(), options.OutputModelsPackage+".") { + return true + } + + // Check if the return type struct contains a type from models package (possibly an enum field or an embedded struct) + if outputFile != OutputFileInterface && q.hasRetType() && q.Ret.IsStruct() { + for _, f := range q.Ret.Struct.Fields { + if strings.HasPrefix(f.Type, options.OutputModelsPackage+".") { + return true + } + } + } + + // Check if the argument type is from models package (possibly an enum) + if !q.Arg.isEmpty() && strings.HasPrefix(q.Arg.Type(), options.OutputModelsPackage+".") { + return true + } + + // Check if the argument struct contains a type from models package (possibly an enum field) + if outputFile != OutputFileInterface && !q.Arg.isEmpty() && q.Arg.IsStruct() { + for _, f := range q.Arg.Struct.Fields { + if strings.HasPrefix(f.Type, options.OutputModelsPackage+".") { + return true + } + } + } + + } + return false + } + if requiresModelsPackageImport() { + pkg[ImportSpec{Path: options.ModelsPackageImportPath}] = struct{}{} + } + return std, pkg } func (i *importer) interfaceImports() fileImports { - std, pkg := buildImports(i.Options, i.Queries, func(name string) bool { + std, pkg := buildImports(i.Options, i.Queries, OutputFileInterface, func(name string) bool { for _, q := range i.Queries { if q.hasRetType() { if usesBatch([]Query{q}) { @@ -272,7 +313,7 @@ func (i *importer) interfaceImports() fileImports { } func (i *importer) modelImports() fileImports { - std, pkg := buildImports(i.Options, nil, i.usesType) + std, pkg := buildImports(i.Options, nil, OutputFileModel, i.usesType) if len(i.Enums) > 0 { std["fmt"] = struct{}{} @@ -311,7 +352,7 @@ func (i *importer) queryImports(filename string) fileImports { } } - std, pkg := buildImports(i.Options, gq, func(name string) bool { + std, pkg := buildImports(i.Options, gq, OutputFileQuery, func(name string) bool { for _, q := range gq { if q.hasRetType() { if q.Ret.EmitStruct() { @@ -412,7 +453,7 @@ func (i *importer) copyfromImports() fileImports { copyFromQueries = append(copyFromQueries, q) } } - std, pkg := buildImports(i.Options, copyFromQueries, func(name string) bool { + std, pkg := buildImports(i.Options, copyFromQueries, OutputFileCopyfrom, func(name string) bool { for _, q := range copyFromQueries { if q.hasRetType() { if strings.HasPrefix(q.Ret.Type(), name) { @@ -447,7 +488,7 @@ func (i *importer) batchImports() fileImports { batchQueries = append(batchQueries, q) } } - std, pkg := buildImports(i.Options, batchQueries, func(name string) bool { + std, pkg := buildImports(i.Options, batchQueries, OutputFileBatch, func(name string) bool { for _, q := range batchQueries { if q.hasRetType() { if q.Ret.EmitStruct() { diff --git a/internal/codegen/golang/opts/options.go b/internal/codegen/golang/opts/options.go index 30a6c2246c..824841f2f4 100644 --- a/internal/codegen/golang/opts/options.go +++ b/internal/codegen/golang/opts/options.go @@ -35,8 +35,11 @@ type Options struct { OutputBatchFileName string `json:"output_batch_file_name,omitempty" yaml:"output_batch_file_name"` OutputDbFileName string `json:"output_db_file_name,omitempty" yaml:"output_db_file_name"` OutputModelsFileName string `json:"output_models_file_name,omitempty" yaml:"output_models_file_name"` + OutputModelsPackage string `json:"output_models_package,omitempty" yaml:"output_models_package"` + ModelsPackageImportPath string `json:"models_package_import_path,omitempty" yaml:"models_package_import_path"` OutputQuerierFileName string `json:"output_querier_file_name,omitempty" yaml:"output_querier_file_name"` OutputCopyfromFileName string `json:"output_copyfrom_file_name,omitempty" yaml:"output_copyfrom_file_name"` + OutputQueryFilesDirectory string `json:"output_query_files_directory,omitempty" yaml:"output_query_files_directory"` OutputFilesSuffix string `json:"output_files_suffix,omitempty" yaml:"output_files_suffix"` InflectionExcludeTableNames []string `json:"inflection_exclude_table_names,omitempty" yaml:"inflection_exclude_table_names"` QueryParameterLimit *int32 `json:"query_parameter_limit,omitempty" yaml:"query_parameter_limit"` @@ -150,6 +153,11 @@ func ValidateOpts(opts *Options) error { if *opts.QueryParameterLimit < 0 { return fmt.Errorf("invalid options: query parameter limit must not be negative") } - + if opts.OutputModelsPackage != "" && opts.ModelsPackageImportPath == "" { + return fmt.Errorf("invalid options: models_package_import_path must be set when output_models_package is used") + } + if opts.ModelsPackageImportPath != "" && opts.OutputModelsPackage == "" { + return fmt.Errorf("invalid options: output_models_package must be set when models_package_import_path is used") + } return nil } diff --git a/internal/codegen/golang/output_file.go b/internal/codegen/golang/output_file.go new file mode 100644 index 0000000000..0e39968b14 --- /dev/null +++ b/internal/codegen/golang/output_file.go @@ -0,0 +1,12 @@ +package golang + +type OutputFile string + +const ( + OutputFileModel OutputFile = "modelFile" + OutputFileQuery OutputFile = "queryFile" + OutputFileDb OutputFile = "dbFile" + OutputFileInterface OutputFile = "interfaceFile" + OutputFileCopyfrom OutputFile = "copyfromFile" + OutputFileBatch OutputFile = "batchFile" +) diff --git a/internal/codegen/golang/postgresql_type.go b/internal/codegen/golang/postgresql_type.go index 4518e1aba6..ee6dd0c09f 100644 --- a/internal/codegen/golang/postgresql_type.go +++ b/internal/codegen/golang/postgresql_type.go @@ -571,17 +571,24 @@ func postgresType(req *plugin.GenerateRequest, options *opts.Options, col *plugi for _, enum := range schema.Enums { if rel.Name == enum.Name && rel.Schema == schema.Name { + enumName := "" if notNull { if schema.Name == req.Catalog.DefaultSchema { - return StructName(enum.Name, options) + enumName = StructName(enum.Name, options) + } else { + enumName = StructName(schema.Name+"_"+enum.Name, options) } - return StructName(schema.Name+"_"+enum.Name, options) } else { if schema.Name == req.Catalog.DefaultSchema { - return "Null" + StructName(enum.Name, options) + enumName = "Null" + StructName(enum.Name, options) + } else { + enumName = "Null" + StructName(schema.Name+"_"+enum.Name, options) } - return "Null" + StructName(schema.Name+"_"+enum.Name, options) } + if options.ModelsPackageImportPath != "" { + return options.OutputModelsPackage + "." + enumName + } + return enumName } } diff --git a/internal/codegen/golang/query.go b/internal/codegen/golang/query.go index 3b4fb2fa1a..6ed3c067d0 100644 --- a/internal/codegen/golang/query.go +++ b/internal/codegen/golang/query.go @@ -88,7 +88,7 @@ func (v QueryValue) Type() string { return v.Typ } if v.Struct != nil { - return v.Struct.Name + return v.Struct.Type() } panic("no type for QueryValue: " + v.Name) } diff --git a/internal/codegen/golang/result.go b/internal/codegen/golang/result.go index 515d0a654f..15ef241a75 100644 --- a/internal/codegen/golang/result.go +++ b/internal/codegen/golang/result.go @@ -83,6 +83,7 @@ func buildStructs(req *plugin.GenerateRequest, options *opts.Options) []Struct { s := Struct{ Table: &plugin.Identifier{Schema: schema.Name, Name: table.Rel.Name}, Name: StructName(structName, options), + Package: options.OutputModelsPackage, Comment: table.Comment, } for _, column := range table.Columns { @@ -146,7 +147,7 @@ func newGoEmbed(embed *plugin.Identifier, structs []Struct, defaultSchema string } return &goEmbed{ - modelType: s.Name, + modelType: s.Type(), modelName: s.Name, fields: fields, } diff --git a/internal/codegen/golang/struct.go b/internal/codegen/golang/struct.go index ed9311800e..e5ae4015c1 100644 --- a/internal/codegen/golang/struct.go +++ b/internal/codegen/golang/struct.go @@ -12,10 +12,18 @@ import ( type Struct struct { Table *plugin.Identifier Name string + Package string Fields []Field Comment string } +func (s Struct) Type() string { + if s.Package != "" { + return s.Package + "." + s.Name + } + return s.Name +} + func StructName(name string, options *opts.Options) string { if rename := options.Rename[name]; rename != "" { return rename diff --git a/internal/codegen/golang/templates/template.tmpl b/internal/codegen/golang/templates/template.tmpl index afd50c01ac..ce603a53e1 100644 --- a/internal/codegen/golang/templates/template.tmpl +++ b/internal/codegen/golang/templates/template.tmpl @@ -156,7 +156,7 @@ type {{.Name}} struct { {{- range .Fields}} {{- if .Comment}} {{comment .Comment}}{{else}} {{- end}} - {{.Name}} {{.Type}} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}} + {{.Name}} {{trimPrefix .Type (printf "%s%s" $.Package ".") }} {{if .Tag}}{{$.Q}}{{.Tag}}{{$.Q}}{{end}} {{- end}} } {{end}} From 638643fc0786fe32d3bf224dd33ccfbb44bcc192 Mon Sep 17 00:00:00 2001 From: berk-karaal Date: Thu, 6 Mar 2025 00:24:35 +0300 Subject: [PATCH 2/3] docs(reference/config): update gen go options --- docs/reference/config.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/reference/config.md b/docs/reference/config.md index e6e690b6a0..aca8f98e2a 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -183,10 +183,16 @@ The `gen` mapping supports the following keys: - Customize the name of the db file. Defaults to `db.go`. - `output_models_file_name`: - Customize the name of the models file. Defaults to `models.go`. +- `output_models_package`: + - Package name of the models file. Used when models file is in a different package. Defaults to value of `package` option. +- `models_package_import_path`: + - Import path of the models package when models file is in a different package. Optional. - `output_querier_file_name`: - Customize the name of the querier file. Defaults to `querier.go`. - `output_copyfrom_file_name`: - Customize the name of the copyfrom file. Defaults to `copyfrom.go`. +- `output_query_files_directory`: + - Directory where the generated query files will be placed. Defaults to the value of `out` option. - `output_files_suffix`: - If specified the suffix will be added to the name of the generated files. - `query_parameter_limit`: From 5121a71a5745bf5932733467fed5e50b091afc83 Mon Sep 17 00:00:00 2001 From: berk-karaal Date: Thu, 6 Mar 2025 01:20:31 +0300 Subject: [PATCH 3/3] docs(howto): add how-to guide for separating models file --- docs/howto/separate-models-file.md | 50 ++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 51 insertions(+) create mode 100644 docs/howto/separate-models-file.md diff --git a/docs/howto/separate-models-file.md b/docs/howto/separate-models-file.md new file mode 100644 index 0000000000..e9cf603b5b --- /dev/null +++ b/docs/howto/separate-models-file.md @@ -0,0 +1,50 @@ +# Separating models file + +By default, sqlc uses a single package to place all the generated code. But you may want to separate +the generated models file into another package for loose coupling purposes in your project. + +To do this, you can use the following configuration: + +```yaml +version: "2" +sql: + - engine: "postgresql" + queries: "queries.sql" + schema: "schema.sql" + gen: + go: + out: "internal/" # Base directory for the generated files. You can also just use "." + sql_package: "pgx/v5" + package: "sqlcrepo" + output_batch_file_name: "db/sqlcrepo/batch.go" + output_db_file_name: "db/sqlcrepo/db.go" + output_querier_file_name: "db/sqlcrepo/querier.go" + output_copyfrom_file_name: "db/sqlcrepo/copyfrom.go" + output_query_files_directory: "db/sqlcrepo/" + output_models_file_name: "business/entities/models.go" + output_models_package: "entities" + models_package_import_path: "example.com/project/module-path/internal/business/entities" +``` + +This configuration will generate files in the `internal/db/sqlcrepo` directory with `sqlcrepo` +package name, except for the models file which will be generated in the `internal/business/entities` +directory. The generated models file will use the package name `entities` and it will be imported in +the other generated files using the given +`"example.com/project/module-path/internal/business/entities"` import path when needed. + +The generated files will look like this: + +``` +my-app/ +├── internal/ +│ ├── db/ +│ │ └── sqlcrepo/ +│ │ ├── db.go +│ │ └── queries.sql.go +│ └── business/ +│ └── entities/ +│ └── models.go +├── queries.sql +├── schema.sql +└── sqlc.yaml +``` diff --git a/docs/index.rst b/docs/index.rst index f914f3ec41..217bf7458a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -66,6 +66,7 @@ code ever again. howto/embedding.md howto/overrides.md howto/rename.md + howto/separate-models-file.md .. toctree:: :maxdepth: 3