Skip to content
This repository was archived by the owner on Sep 9, 2020. It is now read-only.

Commit de0548c

Browse files
sdboyersdboyer
authored
Merge pull request #1570 from sdboyer/fix-prune-opts
Honor per-project prune options correctly
2 parents 69763c4 + 149b895 commit de0548c

13 files changed

+649
-205
lines changed

‎CHANGELOG.md

+7-1
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,16 @@ NEW FEATURES:
44

55
BUG FIXES:
66

7-
* Fix per-project prune option handling ([#1562](https://github.com/golang/dep/pull/1562))
7+
* Fix per-project prune option handling ([#1570](https://github.com/golang/dep/pull/1570))
88

99
IMPROVEMENTS:
1010

11+
# v0.4.1
12+
13+
BUG FIXES:
14+
15+
* Fix per-project prune option handling ([#1570](https://github.com/golang/dep/pull/1570))
16+
1117
# v0.4.0
1218

1319
NEW FEATURES:

‎cmd/dep/init.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
119119
}
120120

121121
// Set default prune options for go-tests and unused-packages
122-
p.Manifest.PruneOptions.PruneOptions = gps.PruneNestedVendorDirs + gps.PruneGoTestFiles + gps.PruneUnusedPackages
122+
p.Manifest.PruneOptions.DefaultOptions = gps.PruneNestedVendorDirs | gps.PruneGoTestFiles | gps.PruneUnusedPackages
123123

124124
if cmd.gopath {
125125
gs := newGopathScanner(ctx, directDeps, sm)

‎cmd/dep/prune.go

+176-3
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,20 @@
55
package main
66

77
import (
8+
"bytes"
89
"flag"
10+
"io/ioutil"
11+
"log"
12+
"os"
13+
"path/filepath"
14+
"sort"
15+
"strings"
916

1017
"github.com/golang/dep"
18+
"github.com/golang/dep/gps"
19+
"github.com/golang/dep/gps/pkgtree"
20+
"github.com/golang/dep/internal/fs"
21+
"github.com/pkg/errors"
1122
)
1223

1324
const pruneShortHelp = `Pruning is now performed automatically by dep ensure.`
@@ -17,20 +28,182 @@ Set prune options in the manifest and it will be applied after every ensure.
1728
dep prune will be removed in a future version of dep, causing this command to exit non-0.
1829
`
1930

20-
type pruneCommand struct{}
31+
type pruneCommand struct {
32+
}
2133

2234
func (cmd *pruneCommand) Name() string { return "prune" }
2335
func (cmd *pruneCommand) Args() string { return "" }
2436
func (cmd *pruneCommand) ShortHelp() string { return pruneShortHelp }
2537
func (cmd *pruneCommand) LongHelp() string { return pruneLongHelp }
26-
func (cmd *pruneCommand) Hidden() bool { return true }
38+
func (cmd *pruneCommand) Hidden() bool { return false }
2739

28-
func (cmd *pruneCommand) Register(fs *flag.FlagSet) {}
40+
func (cmd *pruneCommand) Register(fs *flag.FlagSet) {
41+
}
2942

3043
func (cmd *pruneCommand) Run(ctx *dep.Ctx, args []string) error {
3144
ctx.Out.Printf("Pruning is now performed automatically by dep ensure.\n")
3245
ctx.Out.Printf("Set prune settings in %s and it it will be applied when running ensure.\n", dep.ManifestName)
3346
ctx.Out.Printf("\ndep prune will be removed in a future version, and this command will exit non-0.\nPlease update your scripts.\n")
3447

48+
p, err := ctx.LoadProject()
49+
if err != nil {
50+
return err
51+
}
52+
53+
sm, err := ctx.SourceManager()
54+
if err != nil {
55+
return err
56+
}
57+
sm.UseDefaultSignalHandling()
58+
defer sm.Release()
59+
60+
// While the network churns on ListVersions() requests, statically analyze
61+
// code from the current project.
62+
ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
63+
if err != nil {
64+
return errors.Wrap(err, "analysis of local packages failed: %v")
65+
}
66+
67+
// Set up a solver in order to check the InputHash.
68+
params := p.MakeParams()
69+
params.RootPackageTree = ptree
70+
71+
if ctx.Verbose {
72+
params.TraceLogger = ctx.Err
73+
}
74+
75+
s, err := gps.Prepare(params, sm)
76+
if err != nil {
77+
return errors.Wrap(err, "could not set up solver for input hashing")
78+
}
79+
80+
if p.Lock == nil {
81+
return errors.Errorf("Gopkg.lock must exist for prune to know what files are safe to remove.")
82+
}
83+
84+
if !bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
85+
return errors.Errorf("Gopkg.lock is out of sync; run dep ensure before pruning.")
86+
}
87+
88+
pruneLogger := ctx.Err
89+
if !ctx.Verbose {
90+
pruneLogger = log.New(ioutil.Discard, "", 0)
91+
}
92+
return pruneProject(p, sm, pruneLogger)
93+
}
94+
95+
// pruneProject removes unused packages from a project.
96+
func pruneProject(p *dep.Project, sm gps.SourceManager, logger *log.Logger) error {
97+
td, err := ioutil.TempDir(os.TempDir(), "dep")
98+
if err != nil {
99+
return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor")
100+
}
101+
defer os.RemoveAll(td)
102+
103+
if err := gps.WriteDepTree(td, p.Lock, sm, gps.CascadingPruneOptions{DefaultOptions: gps.PruneNestedVendorDirs}, logger); err != nil {
104+
return err
105+
}
106+
107+
var toKeep []string
108+
for _, project := range p.Lock.Projects() {
109+
projectRoot := string(project.Ident().ProjectRoot)
110+
for _, pkg := range project.Packages() {
111+
toKeep = append(toKeep, filepath.Join(projectRoot, pkg))
112+
}
113+
}
114+
115+
toDelete, err := calculatePrune(td, toKeep, logger)
116+
if err != nil {
117+
return err
118+
}
119+
120+
if len(toDelete) > 0 {
121+
logger.Println("Calculated the following directories to prune:")
122+
for _, d := range toDelete {
123+
logger.Printf(" %s\n", d)
124+
}
125+
} else {
126+
logger.Println("No directories found to prune")
127+
}
128+
129+
if err := deleteDirs(toDelete); err != nil {
130+
return err
131+
}
132+
133+
vpath := filepath.Join(p.AbsRoot, "vendor")
134+
vendorbak := vpath + ".orig"
135+
var failerr error
136+
if _, err := os.Stat(vpath); err == nil {
137+
// Move out the old vendor dir. just do it into an adjacent dir, to
138+
// try to mitigate the possibility of a pointless cross-filesystem
139+
// move with a temp directory.
140+
if _, err := os.Stat(vendorbak); err == nil {
141+
// If the adjacent dir already exists, bite the bullet and move
142+
// to a proper tempdir.
143+
vendorbak = filepath.Join(td, "vendor.orig")
144+
}
145+
failerr = fs.RenameWithFallback(vpath, vendorbak)
146+
if failerr != nil {
147+
goto fail
148+
}
149+
}
150+
151+
// Move in the new one.
152+
failerr = fs.RenameWithFallback(td, vpath)
153+
if failerr != nil {
154+
goto fail
155+
}
156+
157+
os.RemoveAll(vendorbak)
158+
35159
return nil
160+
161+
fail:
162+
fs.RenameWithFallback(vendorbak, vpath)
163+
return failerr
36164
}
165+
166+
func calculatePrune(vendorDir string, keep []string, logger *log.Logger) ([]string, error) {
167+
logger.Println("Calculating prune. Checking the following packages:")
168+
sort.Strings(keep)
169+
toDelete := []string{}
170+
err := filepath.Walk(vendorDir, func(path string, info os.FileInfo, err error) error {
171+
if _, err := os.Lstat(path); err != nil {
172+
return nil
173+
}
174+
if !info.IsDir() {
175+
return nil
176+
}
177+
if path == vendorDir {
178+
return nil
179+
}
180+
181+
name := strings.TrimPrefix(path, vendorDir+string(filepath.Separator))
182+
logger.Printf(" %s", name)
183+
i := sort.Search(len(keep), func(i int) bool {
184+
return name <= keep[i]
185+
})
186+
if i >= len(keep) || !strings.HasPrefix(keep[i], name) {
187+
toDelete = append(toDelete, path)
188+
}
189+
return nil
190+
})
191+
return toDelete, err
192+
}
193+
194+
func deleteDirs(toDelete []string) error {
195+
// sort by length so we delete sub dirs first
196+
sort.Sort(byLen(toDelete))
197+
for _, path := range toDelete {
198+
if err := os.RemoveAll(path); err != nil {
199+
return err
200+
}
201+
}
202+
return nil
203+
}
204+
205+
type byLen []string
206+
207+
func (a byLen) Len() int { return len(a) }
208+
func (a byLen) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
209+
func (a byLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }

‎gps/prune.go

+74-23
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,6 @@ import (
1818
// PruneOptions represents the pruning options used to write the dependecy tree.
1919
type PruneOptions uint8
2020

21-
// PruneProjectOptions is map of prune options per project name.
22-
type PruneProjectOptions map[ProjectRoot]PruneOptions
23-
24-
// RootPruneOptions represents the root prune options for the project.
25-
// It contains the global options and a map of options per project.
26-
type RootPruneOptions struct {
27-
PruneOptions PruneOptions
28-
ProjectOptions PruneProjectOptions
29-
}
30-
3121
const (
3222
// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
3323
PruneNestedVendorDirs PruneOptions = 1 << iota
@@ -41,24 +31,85 @@ const (
4131
PruneGoTestFiles
4232
)
4333

44-
// DefaultRootPruneOptions instantiates a copy of the default root prune options.
45-
func DefaultRootPruneOptions() RootPruneOptions {
46-
return RootPruneOptions{
47-
PruneOptions: PruneNestedVendorDirs,
48-
ProjectOptions: PruneProjectOptions{},
49-
}
34+
// PruneOptionSet represents trinary distinctions for each of the types of
35+
// prune rules (as expressed via PruneOptions): nested vendor directories,
36+
// unused packages, non-go files, and go test files.
37+
//
38+
// The three-way distinction is between "none", "true", and "false", represented
39+
// by uint8 values of 0, 1, and 2, respectively.
40+
//
41+
// This trinary distinction is necessary in order to record, with full fidelity,
42+
// a cascading tree of pruning values, as expressed in CascadingPruneOptions; a
43+
// simple boolean cannot delineate between "false" and "none".
44+
type PruneOptionSet struct {
45+
NestedVendor uint8
46+
UnusedPackages uint8
47+
NonGoFiles uint8
48+
GoTests uint8
49+
}
50+
51+
// CascadingPruneOptions is a set of rules for pruning a dependency tree.
52+
//
53+
// The DefaultOptions are the global default pruning rules, expressed as a
54+
// single PruneOptions bitfield. These global rules will cascade down to
55+
// individual project rules, unless superseded.
56+
type CascadingPruneOptions struct {
57+
DefaultOptions PruneOptions
58+
PerProjectOptions map[ProjectRoot]PruneOptionSet
5059
}
5160

52-
// PruneOptionsFor returns the prune options for the passed project root.
61+
// PruneOptionsFor returns the PruneOptions bits for the given project,
62+
// indicating which pruning rules should be applied to the project's code.
5363
//
54-
// It will return the root prune options if the project does not have specific
55-
// options or if it does not exist in the manifest.
56-
func (o *RootPruneOptions) PruneOptionsFor(pr ProjectRoot) PruneOptions {
57-
if po, ok := o.ProjectOptions[pr]; ok {
58-
return po
64+
// It computes the cascade from default to project-specific options (if any) on
65+
// the fly.
66+
func (o CascadingPruneOptions) PruneOptionsFor(pr ProjectRoot) PruneOptions {
67+
po, has := o.PerProjectOptions[pr]
68+
if !has {
69+
return o.DefaultOptions
70+
}
71+
72+
ops := o.DefaultOptions
73+
if po.NestedVendor != 0 {
74+
if po.NestedVendor == 1 {
75+
ops |= PruneNestedVendorDirs
76+
} else {
77+
ops &^= PruneNestedVendorDirs
78+
}
5979
}
6080

61-
return o.PruneOptions
81+
if po.UnusedPackages != 0 {
82+
if po.UnusedPackages == 1 {
83+
ops |= PruneUnusedPackages
84+
} else {
85+
ops &^= PruneUnusedPackages
86+
}
87+
}
88+
89+
if po.NonGoFiles != 0 {
90+
if po.NonGoFiles == 1 {
91+
ops |= PruneNonGoFiles
92+
} else {
93+
ops &^= PruneNonGoFiles
94+
}
95+
}
96+
97+
if po.GoTests != 0 {
98+
if po.GoTests == 1 {
99+
ops |= PruneGoTestFiles
100+
} else {
101+
ops &^= PruneGoTestFiles
102+
}
103+
}
104+
105+
return ops
106+
}
107+
108+
func defaultCascadingPruneOptions() CascadingPruneOptions {
109+
return CascadingPruneOptions{
110+
DefaultOptions: PruneNestedVendorDirs,
111+
PerProjectOptions: map[ProjectRoot]PruneOptionSet{},
112+
}
62113
}
63114

64115
var (

0 commit comments

Comments
 (0)