X Tutup
Skip to content

Commit 9e34e6e

Browse files
asottileasottile
committed
pre-commit gc
1 parent d7f5c6f commit 9e34e6e

File tree

12 files changed

+412
-116
lines changed

12 files changed

+412
-116
lines changed

‎pre_commit/commands/gc.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
from __future__ import absolute_import
2+
from __future__ import unicode_literals
3+
4+
import os.path
5+
6+
import pre_commit.constants as C
7+
from pre_commit import output
8+
from pre_commit.clientlib import InvalidConfigError
9+
from pre_commit.clientlib import InvalidManifestError
10+
from pre_commit.clientlib import is_local_repo
11+
from pre_commit.clientlib import is_meta_repo
12+
from pre_commit.clientlib import load_config
13+
from pre_commit.clientlib import load_manifest
14+
15+
16+
def _mark_used_repos(store, all_repos, unused_repos, repo):
17+
if is_meta_repo(repo):
18+
return
19+
elif is_local_repo(repo):
20+
for hook in repo['hooks']:
21+
deps = hook.get('additional_dependencies')
22+
unused_repos.discard((
23+
store.db_repo_name(repo['repo'], deps), C.LOCAL_REPO_VERSION,
24+
))
25+
else:
26+
key = (repo['repo'], repo['rev'])
27+
path = all_repos.get(key)
28+
# can't inspect manifest if it isn't cloned
29+
if path is None:
30+
return
31+
32+
try:
33+
manifest = load_manifest(os.path.join(path, C.MANIFEST_FILE))
34+
except InvalidManifestError:
35+
return
36+
else:
37+
unused_repos.discard(key)
38+
by_id = {hook['id']: hook for hook in manifest}
39+
40+
for hook in repo['hooks']:
41+
if hook['id'] not in by_id:
42+
continue
43+
44+
deps = hook.get(
45+
'additional_dependencies',
46+
by_id[hook['id']]['additional_dependencies'],
47+
)
48+
unused_repos.discard((
49+
store.db_repo_name(repo['repo'], deps), repo['rev'],
50+
))
51+
52+
53+
def _gc_repos(store):
54+
configs = store.select_all_configs()
55+
repos = store.select_all_repos()
56+
57+
# delete config paths which do not exist
58+
dead_configs = [p for p in configs if not os.path.exists(p)]
59+
live_configs = [p for p in configs if os.path.exists(p)]
60+
61+
all_repos = {(repo, ref): path for repo, ref, path in repos}
62+
unused_repos = set(all_repos)
63+
for config_path in live_configs:
64+
try:
65+
config = load_config(config_path)
66+
except InvalidConfigError:
67+
dead_configs.append(config_path)
68+
continue
69+
else:
70+
for repo in config['repos']:
71+
_mark_used_repos(store, all_repos, unused_repos, repo)
72+
73+
store.delete_configs(dead_configs)
74+
for db_repo_name, ref in unused_repos:
75+
store.delete_repo(db_repo_name, ref, all_repos[(db_repo_name, ref)])
76+
return len(unused_repos)
77+
78+
79+
def gc(store):
80+
with store.exclusive_lock():
81+
repos_removed = _gc_repos(store)
82+
output.write_line('{} repo(s) removed.'.format(repos_removed))
83+
return 0

‎pre_commit/error_handler.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ def _log_and_exit(msg, exc, formatted):
3232
))
3333
output.write(error_msg)
3434
store = Store()
35-
store.require_created()
3635
log_path = os.path.join(store.directory, 'pre-commit.log')
3736
output.write_line('Check the log at {}'.format(log_path))
3837
with open(log_path, 'wb') as log:

‎pre_commit/main.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from pre_commit import git
1212
from pre_commit.commands.autoupdate import autoupdate
1313
from pre_commit.commands.clean import clean
14+
from pre_commit.commands.gc import gc
1415
from pre_commit.commands.install_uninstall import install
1516
from pre_commit.commands.install_uninstall import install_hooks
1617
from pre_commit.commands.install_uninstall import uninstall
@@ -176,6 +177,11 @@ def main(argv=None):
176177
)
177178
_add_color_option(clean_parser)
178179
_add_config_option(clean_parser)
180+
181+
gc_parser = subparsers.add_parser('gc', help='Clean unused cached repos.')
182+
_add_color_option(gc_parser)
183+
_add_config_option(gc_parser)
184+
179185
autoupdate_parser = subparsers.add_parser(
180186
'autoupdate',
181187
help="Auto-update pre-commit config to the latest repos' versions.",
@@ -251,9 +257,11 @@ def main(argv=None):
251257
with error_handler(), logging_handler(args.color):
252258
_adjust_args_and_chdir(args)
253259

254-
store = Store()
255260
git.check_for_cygwin_mismatch()
256261

262+
store = Store()
263+
store.mark_config_used(args.config)
264+
257265
if args.command == 'install':
258266
return install(
259267
args.config, store,
@@ -267,6 +275,8 @@ def main(argv=None):
267275
return uninstall(hook_type=args.hook_type)
268276
elif args.command == 'clean':
269277
return clean(store)
278+
elif args.command == 'gc':
279+
return gc(store)
270280
elif args.command == 'autoupdate':
271281
if args.tags_only:
272282
logger.warning('--tags-only is the default')

‎pre_commit/store.py‎

Lines changed: 91 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from pre_commit.util import clean_path_on_failure
1414
from pre_commit.util import cmd_output
1515
from pre_commit.util import resource_text
16+
from pre_commit.util import rmtree
1617

1718

1819
logger = logging.getLogger('pre_commit')
@@ -33,10 +34,43 @@ def _get_default_directory():
3334

3435
class Store(object):
3536
get_default_directory = staticmethod(_get_default_directory)
36-
__created = False
3737

3838
def __init__(self, directory=None):
3939
self.directory = directory or Store.get_default_directory()
40+
self.db_path = os.path.join(self.directory, 'db.db')
41+
42+
if not os.path.exists(self.directory):
43+
os.makedirs(self.directory)
44+
with io.open(os.path.join(self.directory, 'README'), 'w') as f:
45+
f.write(
46+
'This directory is maintained by the pre-commit project.\n'
47+
'Learn more: https://github.com/pre-commit/pre-commit\n',
48+
)
49+
50+
if os.path.exists(self.db_path):
51+
return
52+
with self.exclusive_lock():
53+
# Another process may have already completed this work
54+
if os.path.exists(self.db_path): # pragma: no cover (race)
55+
return
56+
# To avoid a race where someone ^Cs between db creation and
57+
# execution of the CREATE TABLE statement
58+
fd, tmpfile = tempfile.mkstemp(dir=self.directory)
59+
# We'll be managing this file ourselves
60+
os.close(fd)
61+
with self.connect(db_path=tmpfile) as db:
62+
db.executescript(
63+
'CREATE TABLE repos ('
64+
' repo TEXT NOT NULL,'
65+
' ref TEXT NOT NULL,'
66+
' path TEXT NOT NULL,'
67+
' PRIMARY KEY (repo, ref)'
68+
');',
69+
)
70+
self._create_config_table_if_not_exists(db)
71+
72+
# Atomic file move
73+
os.rename(tmpfile, self.db_path)
4074

4175
@contextlib.contextmanager
4276
def exclusive_lock(self):
@@ -46,62 +80,30 @@ def blocked_cb(): # pragma: no cover (tests are single-process)
4680
with file_lock.lock(os.path.join(self.directory, '.lock'), blocked_cb):
4781
yield
4882

49-
def _write_readme(self):
50-
with io.open(os.path.join(self.directory, 'README'), 'w') as readme:
51-
readme.write(
52-
'This directory is maintained by the pre-commit project.\n'
53-
'Learn more: https://github.com/pre-commit/pre-commit\n',
54-
)
55-
56-
def _write_sqlite_db(self):
57-
# To avoid a race where someone ^Cs between db creation and execution
58-
# of the CREATE TABLE statement
59-
fd, tmpfile = tempfile.mkstemp(dir=self.directory)
60-
# We'll be managing this file ourselves
61-
os.close(fd)
83+
@contextlib.contextmanager
84+
def connect(self, db_path=None):
85+
db_path = db_path or self.db_path
6286
# sqlite doesn't close its fd with its contextmanager >.<
6387
# contextlib.closing fixes this.
6488
# See: https://stackoverflow.com/a/28032829/812183
65-
with contextlib.closing(sqlite3.connect(tmpfile)) as db:
66-
db.executescript(
67-
'CREATE TABLE repos ('
68-
' repo TEXT NOT NULL,'
69-
' ref TEXT NOT NULL,'
70-
' path TEXT NOT NULL,'
71-
' PRIMARY KEY (repo, ref)'
72-
');',
73-
)
89+
with contextlib.closing(sqlite3.connect(db_path)) as db:
90+
# this creates a transaction
91+
with db:
92+
yield db
7493

75-
# Atomic file move
76-
os.rename(tmpfile, self.db_path)
77-
78-
def _create(self):
79-
if not os.path.exists(self.directory):
80-
os.makedirs(self.directory)
81-
self._write_readme()
82-
83-
if os.path.exists(self.db_path):
84-
return
85-
with self.exclusive_lock():
86-
# Another process may have already completed this work
87-
if os.path.exists(self.db_path): # pragma: no cover (race)
88-
return
89-
self._write_sqlite_db()
90-
91-
def require_created(self):
92-
"""Require the pre-commit file store to be created."""
93-
if not self.__created:
94-
self._create()
95-
self.__created = True
94+
@classmethod
95+
def db_repo_name(cls, repo, deps):
96+
if deps:
97+
return '{}:{}'.format(repo, ','.join(sorted(deps)))
98+
else:
99+
return repo
96100

97101
def _new_repo(self, repo, ref, deps, make_strategy):
98-
self.require_created()
99-
if deps:
100-
repo = '{}:{}'.format(repo, ','.join(sorted(deps)))
102+
repo = self.db_repo_name(repo, deps)
101103

102104
def _get_result():
103105
# Check if we already exist
104-
with sqlite3.connect(self.db_path) as db:
106+
with self.connect() as db:
105107
result = db.execute(
106108
'SELECT path FROM repos WHERE repo = ? AND ref = ?',
107109
(repo, ref),
@@ -125,7 +127,7 @@ def _get_result():
125127
make_strategy(directory)
126128

127129
# Update our db with the created repo
128-
with sqlite3.connect(self.db_path) as db:
130+
with self.connect() as db:
129131
db.execute(
130132
'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
131133
[repo, ref, directory],
@@ -175,6 +177,43 @@ def _git_cmd(*args):
175177
'local', C.LOCAL_REPO_VERSION, deps, make_local_strategy,
176178
)
177179

178-
@property
179-
def db_path(self):
180-
return os.path.join(self.directory, 'db.db')
180+
def _create_config_table_if_not_exists(self, db):
181+
db.executescript(
182+
'CREATE TABLE IF NOT EXISTS configs ('
183+
' path TEXT NOT NULL,'
184+
' PRIMARY KEY (path)'
185+
');',
186+
)
187+
188+
def mark_config_used(self, path):
189+
path = os.path.realpath(path)
190+
# don't insert config files that do not exist
191+
if not os.path.exists(path):
192+
return
193+
with self.connect() as db:
194+
# TODO: eventually remove this and only create in _create
195+
self._create_config_table_if_not_exists(db)
196+
db.execute('INSERT OR IGNORE INTO configs VALUES (?)', (path,))
197+
198+
def select_all_configs(self):
199+
with self.connect() as db:
200+
self._create_config_table_if_not_exists(db)
201+
rows = db.execute('SELECT path FROM configs').fetchall()
202+
return [path for path, in rows]
203+
204+
def delete_configs(self, configs):
205+
with self.connect() as db:
206+
rows = [(path,) for path in configs]
207+
db.executemany('DELETE FROM configs WHERE path = ?', rows)
208+
209+
def select_all_repos(self):
210+
with self.connect() as db:
211+
return db.execute('SELECT repo, ref, path from repos').fetchall()
212+
213+
def delete_repo(self, db_repo_name, ref, path):
214+
with self.connect() as db:
215+
db.execute(
216+
'DELETE FROM repos WHERE repo = ? and ref = ?',
217+
(db_repo_name, ref),
218+
)
219+
rmtree(path)

‎testing/fixtures.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ def modify_config(path='.', commit=True):
8282
git_commit(msg=modify_config.__name__, cwd=path)
8383

8484

85-
def config_with_local_hooks():
85+
def sample_local_config():
8686
return {
8787
'repo': 'local',
8888
'hooks': [{
@@ -94,6 +94,10 @@ def config_with_local_hooks():
9494
}
9595

9696

97+
def sample_meta_config():
98+
return {'repo': 'meta', 'hooks': [{'id': 'check-useless-excludes'}]}
99+
100+
97101
def make_config_from_repo(repo_path, rev=None, hooks=None, check=True):
98102
manifest = load_manifest(os.path.join(repo_path, C.MANIFEST_FILE))
99103
config = {

‎tests/clientlib_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from pre_commit.clientlib import MigrateShaToRev
1212
from pre_commit.clientlib import validate_config_main
1313
from pre_commit.clientlib import validate_manifest_main
14-
from testing.fixtures import config_with_local_hooks
14+
from testing.fixtures import sample_local_config
1515
from testing.util import get_resource_path
1616

1717

@@ -94,7 +94,7 @@ def test_config_valid(config_obj, expected):
9494

9595

9696
def test_local_hooks_with_rev_fails():
97-
config_obj = {'repos': [config_with_local_hooks()]}
97+
config_obj = {'repos': [sample_local_config()]}
9898
config_obj['repos'][0]['rev'] = 'foo'
9999
with pytest.raises(cfgv.ValidationError):
100100
cfgv.validate(config_obj, CONFIG_SCHEMA)

0 commit comments

Comments
 (0)
X Tutup