WIP: Optimize for big repository on repo home page to make it load less than 1s and /main less than 8s

custom/conf/app.ini.sample

@@ -455,6 +455,24 @@ HOST =
 ; Setting it to 0 disables caching
 ITEM_TTL = 16h
 
+; Last commit cache
+[cache.last_commit]
+; Cache type could be empty, "default", "memory", "boltdb", "redis", empty to disable this cache
+TYPE = boltdb
+; repos which have less than commits number will enable this cache, 0 will always cache if `TYPE` is not empty
+ENABLE_MIN_COMMIT_COUNT = 1000
+; Cache dir when type is boltdb, when `TYPE` is `redis`, will be `addrs=127.0.0.1 password= db=0`
+CONN_STR = data/last_commit_cache_boltdb
+
+; Ls tree cache
+[cache.ls_tree]
+; Cache type could be empty, "default", "memory", "boltdb", "redis", empty to disable this cache
+TYPE = boltdb
+; repos which have less than commits number will enable this cache, 0 will always cache if `TYPE` is not empty
+ENABLE_MIN_COMMIT_COUNT = 1000
+; Cache dir when type is boltdb, when `TYPE` is `redis`, will be `addrs=127.0.0.1 password= db=0`
+CONN_STR = data/ls_tree_cache_boltdb
+
 [session]
 ; Either "memory", "file", or "redis", default is "memory"
 PROVIDER = memory

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -258,6 +258,18 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
    - Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
    - Memache: `127.0.0.1:9090;127.0.0.1:9091`
 
+## Cache - LastCommitCache settings (`cache.last_commit`)
+
+- `TYPE`: **boltdb**: Cache type could be empty, `default`, `memory`, `boltdb`, `redis`, empty to disable this cache
+- `ENABLE_MIN_COMMIT_COUNT`: **1000**: repos which have less than commits number will enable this cache, 0 will always cache if `TYPE` is not empty
+- `CONN_STR`: **data/last_commit_cache_boltdb**: Cache dir when type is boltdb, when `TYPE` is `redis`, will be `addrs=127.0.0.1 password= db=0`
+
+## Cache - LsTreeCache settings (`cache.ls_tree`)
+
+- `TYPE`: **boltdb**: Cache type could be empty, `default`, `memory`, `boltdb`, `redis`, empty to disable this cache
+- `ENABLE_MIN_COMMIT_COUNT`: **1000**: repos which have less than commits number will enable this cache, 0 will always cache if `TYPE` is not empty
+- `CONN_STR`: **data/ls_tree_cache_boltdb**: Cache dir when type is boltdb, when `TYPE` is `redis`, will be `addrs=127.0.0.1 password= db=0`
+
 ## Session (`session`)
 
 - `PROVIDER`: **memory**: Session engine provider \[memory, file, redis, mysql, couchbase, memcache, nodb, postgres\].
@@ -326,6 +338,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `GC_ARGS`: **\<empty\>**: Arguments for command `git gc`, e.g. `--aggressive --auto`. See more on http://git-scm.com/docs/git-gc/
 
 ## Git - Timeout settings (`git.timeout`)
+
 - `DEFAUlT`: **360**: Git operations default timeout seconds.
 - `MIGRATE`: **600**: Migrate external repositories timeout seconds.
 - `MIRROR`: **300**: Mirror external repositories timeout seconds.

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -147,6 +147,19 @@ menu:
     - Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
     - Memache: `127.0.0.1:9090;127.0.0.1:9091`
 
+
+## Cache - LastCommitCache settings (`cache.last_commit`)
+
+- `TYPE`: **boltdb**: 缓存类型，可以为空，`default`， `memory`， `boltdb`， `redis`。为空则禁用此缓存。
+- `ENABLE_MIN_COMMIT_COUNT`: **1000**: 缓存类型，可以为空，`default`， `memory`， `boltdb`， `redis`。为空则禁用此缓存。
+- `CONN_STR`: **data/last_commit_cache_boltdb**: 如果缓存类型为 `boltdb` 则为缓存文件路径，如果为 `redis`，则为 `addrs=127.0.0.1 password= db=0`
+
+## Cache - LsTreeCache settings (`cache.ls_tree`)
+
+- `TYPE`: **boltdb**: 缓存类型，可以为空，`default`， `memory`， `boltdb`， `redis`。为空则禁用此缓存。
+- `ENABLE_MIN_COMMIT_COUNT`: **1000**: 缓存类型，可以为空，`default`， `memory`， `boltdb`， `redis`。为空则禁用此缓存。
+- `CONN_STR`: **data/ls_tree_cache_boltdb**: 如果缓存类型为 `boltdb` 则为缓存文件路径，如果为 `redis`，则为 `addrs=127.0.0.1 password= db=0`
+
 ## Session (`session`)
 
 - `PROVIDER`: Session 内容存储方式，可选 `memory`, `file`, `redis` 或 `mysql`。

modules/cache/cache.go

@@ -13,16 +13,17 @@ import (
 	mc "github.com/go-macaron/cache"
 )
 
-var conn mc.Cache
+// Cache defines the cache provider
+var Cache mc.Cache
 
 // NewContext start cache service
 func NewContext() error {
-	if setting.CacheService == nil || conn != nil {
+	if setting.CacheService.Adapter == "" || Cache != nil {
 		return nil
 	}
 
 	var err error
-	conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
+	Cache, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
 		Adapter:       setting.CacheService.Adapter,
 		AdapterConfig: setting.CacheService.Conn,
 		Interval:      setting.CacheService.Interval,
@@ -32,20 +33,20 @@ func NewContext() error {
 
 // GetInt returns key value from cache with callback when no key exists in cache
 func GetInt(key string, getFunc func() (int, error)) (int, error) {
-	if conn == nil || setting.CacheService.TTL == 0 {
+	if Cache == nil || setting.CacheService.TTL == 0 {
 		return getFunc()
 	}
-	if !conn.IsExist(key) {
+	if !Cache.IsExist(key) {
 		var (
 			value int
 			err   error
 		)
 		if value, err = getFunc(); err != nil {
 			return value, err
 		}
-		conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+		Cache.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
 	}
-	switch value := conn.Get(key).(type) {
+	switch value := Cache.Get(key).(type) {
 	case int:
 		return value, nil
 	case string:
@@ -61,20 +62,20 @@ func GetInt(key string, getFunc func() (int, error)) (int, error) {
 
 // GetInt64 returns key value from cache with callback when no key exists in cache
 func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
-	if conn == nil || setting.CacheService.TTL == 0 {
+	if Cache == nil || setting.CacheService.TTL == 0 {
 		return getFunc()
 	}
-	if !conn.IsExist(key) {
+	if !Cache.IsExist(key) {
 		var (
 			value int64
 			err   error
 		)
 		if value, err = getFunc(); err != nil {
 			return value, err
 		}
-		conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+		Cache.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
 	}
-	switch value := conn.Get(key).(type) {
+	switch value := Cache.Get(key).(type) {
 	case int64:
 		return value, nil
 	case string:
@@ -90,8 +91,8 @@ func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
 
 // Remove key from cache
 func Remove(key string) {
-	if conn == nil {
+	if Cache == nil {
 		return
 	}
-	conn.Delete(key)
+	Cache.Delete(key)
 }

modules/cache/lastcommit/boltdb.go

@@ -0,0 +1,86 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lastcommit
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"code.gitea.io/git"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+var (
+	_ git.LastCommitCache = &BoltDBCache{}
+)
+
+// BoltDBCache implements git.LastCommitCache interface to save the last commits on boltdb
+type BoltDBCache struct {
+	cacheDir string
+	bucket   []byte
+	db       *bolt.DB
+}
+
+// NewBoltDBCache creates a boltdb cache
+func NewBoltDBCache(cacheDir string) (*BoltDBCache, error) {
+	db, err := bolt.Open(cacheDir, 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var bucket = []byte("default")
+	err = db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucket)
+		if err != nil {
+			return fmt.Errorf("create bucket: %s", err)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &BoltDBCache{
+		cacheDir: cacheDir,
+		bucket:   bucket,
+		db:       db,
+	}, nil
+}
+
+// Get implements git.LastCommitCache
+func (c *BoltDBCache) Get(repoPath, ref, entryPath string) (*git.Commit, error) {
+	var commit git.Commit
+	var found bool
+	err := c.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(c.bucket)
+		v := b.Get([]byte(getKey(repoPath, ref, entryPath)))
+		if v == nil || len(v) <= 0 {
+			return nil
+		}
+		found = true
+		return json.Unmarshal(v, &commit)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if found {
+		return &commit, nil
+	}
+	return nil, nil
+}
+
+// Put implements git.LastCommitCache
+func (c *BoltDBCache) Put(repoPath, ref, entryPath string, commit *git.Commit) error {
+	err := c.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(c.bucket)
+		v, err := json.Marshal(commit)
+		if err != nil {
+			return err
+		}
+		return b.Put([]byte(getKey(repoPath, ref, entryPath)), v)
+	})
+	return err
+}

modules/cache/lastcommit/cache.go

@@ -0,0 +1,34 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lastcommit
+
+import (
+	"fmt"
+
+	"code.gitea.io/git"
+
+	"github.com/go-macaron/cache"
+)
+
+type lastCommitCache struct {
+	mc      cache.Cache
+	timeout int64 // seconds
+}
+
+func getKey(repoPath, ref, entryPath string) string {
+	return fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)
+}
+
+func (l *lastCommitCache) Get(repoPath, ref, entryPath string) (*git.Commit, error) {
+	res := l.mc.Get(getKey(repoPath, ref, entryPath))
+	if res == nil {
+		return nil, nil
+	}
+	return res.(*git.Commit), nil
+}
+
+func (l *lastCommitCache) Put(repoPath, ref, entryPath string, commit *git.Commit) error {
+	return l.mc.Put(getKey(repoPath, ref, entryPath), commit, l.timeout)
+}

modules/cache/lastcommit/init.go

@@ -0,0 +1,56 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lastcommit
+
+import (
+	"fmt"
+	"time"
+
+	"code.gitea.io/git"
+	"code.gitea.io/gitea/modules/cache"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var (
+	// Cache defines globally last commit cache object
+	Cache git.LastCommitCache
+)
+
+// NewContext init
+func NewContext() error {
+	var err error
+	switch setting.CacheService.LastCommit.Type {
+	case "default":
+		if cache.Cache != nil {
+			Cache = &lastCommitCache{
+				mc:      cache.Cache,
+				timeout: int64(setting.CacheService.TTL / time.Second),
+			}
+		} else {
+			log.Warn("Last Commit Cache Enabled but Cache Service not Configed Well")
+			return nil
+		}
+	case "memory":
+		Cache = &MemoryCache{}
+	case "boltdb":
+		Cache, err = NewBoltDBCache(setting.CacheService.LastCommit.ConnStr)
+	case "redis":
+		addrs, pass, dbIdx, err := parseConnStr(setting.CacheService.LastCommit.ConnStr)
+		if err != nil {
+			return err
+		}
+
+		Cache, err = NewRedisCache(addrs, pass, dbIdx)
+	case "":
+		return nil
+	default:
+		return fmt.Errorf("Unsupported last commit type: %s", setting.CacheService.LastCommit.Type)
+	}
+	if err == nil {
+		log.Info("Last Commit Cache %s Enabled", setting.CacheService.LastCommit.Type)
+	}
+	return err
+}