From aa45b304840ce3a67ff7a6a06cd9db94d26d1366 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 1 Mar 2018 12:42:20 -0800 Subject: [PATCH 01/45] add workspace that clones repo --- .gitignore | 1 + Pipfile | 1 + Pipfile.lock | 23 ++++++++++++- langserver/clone_workspace.py | 65 +++++++++++++++++++++++++++++++++++ langserver/config.py | 1 + setup.cfg | 2 ++ test/test_clone_workspace.py | 24 +++++++++++++ 7 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 langserver/clone_workspace.py create mode 100644 test/test_clone_workspace.py diff --git a/.gitignore b/.gitignore index dd248a3..93417db 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ __pycache__/ src .cache/ python-langserver-cache/ +python-cloned-projects-cache/ \ No newline at end of file diff --git a/Pipfile b/Pipfile index 0a728ec..1bd4b1b 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ jedi = {git = "git://github.com/sourcegraph/jedi.git", editable = true, ref = "9 requirements = {git = "git://github.com/sourcegraph/requirements-parser.git", editable = true, ref = "69f1a9cb916b2995843c3ea9b988da46c9dd65c7"} opentracing = "*" lightstep = "*" +"delegator.py" = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 906b3c5..8362b8e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d1f97bcc1910b5a1f37dfe415f819ab2762d99a804119cbe1ab9480bac33e8ab" + "sha256": "a87c841063a059630cc78a587661ae1d78009bfff052cfb964b88571f2f75be1" }, "host-environment-markers": { "implementation_name": "cpython", @@ -35,6 +35,13 @@ ], "version": "==2.2.1" }, + "delegator.py": { + "hashes": [ + "sha256:58f3ea6fe36680e1d828e2e66e52844b826f186409dfee4436e42351b0e699fe", + "sha256:2d46966a7f484d271b09e2646eae1e9acadc4fdf2cb760c142f073e81c927d8d" + ], + "version": "==0.1.0" + }, "jedi": { "editable": true, "git": "git://github.com/sourcegraph/jedi.git", @@ -59,6 +66,13 @@ ], "version": "==1.3.0" }, + "pexpect": { + "hashes": [ + "sha256:6ff881b07aff0cb8ec02055670443f784434395f90c3285d2ae470f921ade52a", + "sha256:67b85a1565968e3d5b5e7c9283caddc90c3947a2625bed1905be27bd5a03e47d" + ], + "version": "==4.4.0" + }, "protobuf": { "hashes": [ "sha256:11788df3e176f44e0375fe6361342d7258a457b346504ea259a21b77ffc18a90", @@ -72,6 +86,13 @@ ], "version": "==3.5.1" }, + "ptyprocess": { + "hashes": [ + "sha256:e8c43b5eee76b2083a9badde89fd1bbce6c8942d1045146e100b7b5e014f4f1a", + "sha256:e64193f0047ad603b71f202332ab5527c5e52aa7c8b609704fc28c0dc20c4365" + ], + "version": "==0.5.2" + }, "requirements": { "editable": true, "git": "git://github.com/sourcegraph/requirements-parser.git", diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py new file mode 100644 index 0000000..308a8ea --- /dev/null +++ b/langserver/clone_workspace.py @@ -0,0 +1,65 @@ +import logging +import os +import os.path +from .config import GlobalConfig +from .fs import FileSystem +from shutil import rmtree + +log = logging.getLogger(__name__) + + +class CloneWorkspace: + + def __init__(self, fs: FileSystem, project_root: str, + original_root_path: str= ""): + + self.PROJECT_ROOT = project_root + self.repo = None + self.hash = None + + if original_root_path.startswith( + "git://") and "?" in original_root_path: + repo_and_hash = original_root_path.split("?") + self.repo = repo_and_hash[0] + original_root_path = self.repo + self.hash = repo_and_hash[1] + + if original_root_path.startswith("git://"): + original_root_path = original_root_path[6:] + + # turn the original root path into something that can be used as a + # file/path name or cache key + self.key = original_root_path.replace("/", ".").replace("\\", ".") + if self.hash: + self.key = ".".join((self.key, self.hash)) + + # TODO: allow different Python versions per project/workspace + self.PYTHON_PATH = GlobalConfig.PYTHON_PATH + self.CLONED_PROJECT_PATH = os.path.join( + GlobalConfig.CLONED_PROJECT_PATH, self.key) + log.debug("Setting Python path to %s", self.PYTHON_PATH) + log.debug("Setting Cloned Project path to %s", + self.CLONED_PROJECT_PATH) + + self.fs = fs + + def clone_project(self): + """ + Clones the project from the provided filesystem into the local + cache + """ + all_files = self.fs.walk(self.PROJECT_ROOT) + for file_path, file_contents in self.fs.batch_open(all_files, parent_span=None): + # strip the leading '/' so that we can join it properly + file_path = os.path.relpath(file_path, "/") + + cache_file_path = os.path.join(self.CLONED_PROJECT_PATH, file_path) + + os.makedirs(os.path.dirname(cache_file_path), exist_ok=True) + + with open(cache_file_path, "w") as f: + f.write(file_contents) + + def cleanup(self): + log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) + rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) diff --git a/langserver/config.py b/langserver/config.py index 797986a..475cd6f 100644 --- a/langserver/config.py +++ b/langserver/config.py @@ -7,5 +7,6 @@ class GlobalConfig: PYTHON_PATH = distutils.sysconfig.get_python_lib(standard_lib=True) PACKAGES_PARENT = "python-langserver-cache" + CLONED_PROJECT_PATH = "python-cloned-projects-cache" STDLIB_REPO_URL = "git://github.com/python/cpython" STDLIB_SRC_PATH = "Lib" diff --git a/setup.cfg b/setup.cfg index ad65b3b..b2b68f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,7 @@ exclude = __pycache__, test_langserver.py, *python-langserver-cache*, + *python-cloned-projects-cache*, # don't run flake8 on repo submodules test/repos/* max-line-length=100 @@ -14,6 +15,7 @@ exclude = __pycache__, test_langserver.py, *python-langserver-cache*, + *python-cloned-projects-cache*, # don't run flake8 on repo submodules test/repos/* max-line-length=100 diff --git a/test/test_clone_workspace.py b/test/test_clone_workspace.py new file mode 100644 index 0000000..8bd81c8 --- /dev/null +++ b/test/test_clone_workspace.py @@ -0,0 +1,24 @@ +from langserver.clone_workspace import CloneWorkspace +from langserver.fs import TestFileSystem +import delegator +import pytest + + +@pytest.fixture() +def test_data(): + repoPath = "repos/fizzbuzz_service" + workspace = CloneWorkspace(TestFileSystem(repoPath), repoPath, repoPath) + yield (workspace, repoPath) + workspace.cleanup() + + +class TestCloningWorkspace: + def test_clone(self, test_data): + workspace, repoPath = test_data + workspace.clone_project() + + c = delegator.run("diff -r {} {}".format(repoPath, + workspace.CLONED_PROJECT_PATH)) + assert c.out == "" + assert c.err == "" + assert c.return_code == 0 From 56cb4e938727d4e7fd8da98c7071561028a86a64 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 1 Mar 2018 15:42:39 -0800 Subject: [PATCH 02/45] fs: don't yield non-text files --- langserver/fs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/langserver/fs.py b/langserver/fs.py index 9e3e958..e507797 100644 --- a/langserver/fs.py +++ b/langserver/fs.py @@ -4,7 +4,7 @@ import opentracing from typing import List - +import mimetypes from .jsonrpc import JSONRPC2Connection @@ -212,7 +212,8 @@ def _walk(self, top: str): if os.path.isdir(e): dirs.append(os.path.relpath(e, self.root)) else: - files.append(os.path.relpath(e, self.root)) + if mimetypes.guess_type(e)[0].startswith("text/"): + files.append(os.path.relpath(e, self.root)) yield from files for d in dirs: yield from self._walk(os.path.join(self.root, d)) From b530d1c024283aabe6829b43e2fef30080638237 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 1 Mar 2018 16:19:54 -0800 Subject: [PATCH 03/45] wip - work on making fizzbuzz tests pass --- Makefile | 4 +- Pipfile | 3 +- Pipfile.lock | 65 ++- langserver/clone_workspace.py | 120 ++++- langserver/jedi.py | 145 +----- langserver/langserver.py | 7 +- test/test_clone_workspace.py | 1 - test/test_fizzbuzz.py | 880 +++++++++++++++++----------------- 8 files changed, 625 insertions(+), 600 deletions(-) diff --git a/Makefile b/Makefile index e71d8b0..508cedb 100644 --- a/Makefile +++ b/Makefile @@ -8,5 +8,5 @@ lint: pipenv run flake8 test: - pipenv run pytest test_langserver.py - cd ./test && pipenv run pytest test_*.py + # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_fizzbuzz.py diff --git a/Pipfile b/Pipfile index 1bd4b1b..48fbf2f 100644 --- a/Pipfile +++ b/Pipfile @@ -7,11 +7,12 @@ name = "pypi" [packages] -jedi = {git = "git://github.com/sourcegraph/jedi.git", editable = true, ref = "9a3e7256df2e6099207fd7289141885ec17ebec7"} requirements = {git = "git://github.com/sourcegraph/requirements-parser.git", editable = true, ref = "69f1a9cb916b2995843c3ea9b988da46c9dd65c7"} opentracing = "*" lightstep = "*" "delegator.py" = "*" +pipenv = "*" +jedi = {git = "git://github.com/sourcegraph/jedi.git", editable = true, ref = "dee138a06ce5568bfda1f88d77184223c5cf5b8d"} [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 8362b8e..1b54c75 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a87c841063a059630cc78a587661ae1d78009bfff052cfb964b88571f2f75be1" + "sha256": "a60044d027ec228213683ce4cc0cfc7246552d4a800ad9c284756dea51651027" }, "host-environment-markers": { "implementation_name": "cpython", @@ -35,6 +35,20 @@ ], "version": "==2.2.1" }, + "certifi": { + "hashes": [ + "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", + "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" + ], + "version": "==2018.1.18" + }, + "chardet": { + "hashes": [ + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" + ], + "version": "==3.0.4" + }, "delegator.py": { "hashes": [ "sha256:58f3ea6fe36680e1d828e2e66e52844b826f186409dfee4436e42351b0e699fe", @@ -42,10 +56,17 @@ ], "version": "==0.1.0" }, + "idna": { + "hashes": [ + "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", + "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" + ], + "version": "==2.6" + }, "jedi": { "editable": true, "git": "git://github.com/sourcegraph/jedi.git", - "ref": "9a3e7256df2e6099207fd7289141885ec17ebec7" + "ref": "dee138a06ce5568bfda1f88d77184223c5cf5b8d" }, "jsonpickle": { "hashes": [ @@ -66,6 +87,13 @@ ], "version": "==1.3.0" }, + "pew": { + "hashes": [ + "sha256:a5256586e169199cb2213225261b6bdd7b657c309dcf4a02b61994308b313d4d", + "sha256:b8312728526c9010295c88215c95a1b1731fdbd1a568f728e069932bd0545611" + ], + "version": "==1.1.2" + }, "pexpect": { "hashes": [ "sha256:6ff881b07aff0cb8ec02055670443f784434395f90c3285d2ae470f921ade52a", @@ -73,6 +101,12 @@ ], "version": "==4.4.0" }, + "pipenv": { + "hashes": [ + "sha256:ce6dbb305fb1f262dba0dcb50c06591e4d146f7bfe079cc9f0ce3f89c7516ae9" + ], + "version": "==10.1.2" + }, "protobuf": { "hashes": [ "sha256:11788df3e176f44e0375fe6361342d7258a457b346504ea259a21b77ffc18a90", @@ -93,6 +127,13 @@ ], "version": "==0.5.2" }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "version": "==2.18.4" + }, "requirements": { "editable": true, "git": "git://github.com/sourcegraph/requirements-parser.git", @@ -115,6 +156,26 @@ "sha256:b7f6c09155321169af03f9fb20dc15a4a0c7481e7c334a5ba8f7f0d864633209" ], "version": "==0.10.0" + }, + "urllib3": { + "hashes": [ + "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", + "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" + ], + "version": "==1.22" + }, + "virtualenv": { + "hashes": [ + "sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1fc9fbb7b481e5c85b0", + "sha256:02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a" + ], + "version": "==15.1.0" + }, + "virtualenv-clone": { + "hashes": [ + "sha256:6b3be5cab59e455f08c9eda573d23006b7d6fb41fae974ddaa2b275c93cc4405" + ], + "version": "==0.2.6" } }, "develop": { diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 308a8ea..fe275db 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -4,6 +4,8 @@ from .config import GlobalConfig from .fs import FileSystem from shutil import rmtree +import delegator +import json log = logging.getLogger(__name__) @@ -43,23 +45,119 @@ def __init__(self, fs: FileSystem, project_root: str, self.fs = fs - def clone_project(self): - """ - Clones the project from the provided filesystem into the local - cache - """ + # Clone the project from the provided filesystem into the local + # cache all_files = self.fs.walk(self.PROJECT_ROOT) - for file_path, file_contents in self.fs.batch_open(all_files, parent_span=None): - # strip the leading '/' so that we can join it properly - file_path = os.path.relpath(file_path, "/") - - cache_file_path = os.path.join(self.CLONED_PROJECT_PATH, file_path) + for file_path in all_files: + cache_file_path = self.to_cache_path(file_path) + if os.path.exists(cache_file_path): + continue os.makedirs(os.path.dirname(cache_file_path), exist_ok=True) - + file_contents = self.fs.open(file_path) with open(cache_file_path, "w") as f: f.write(file_contents) + self.ensure_venv_created() + self.VENV_LOCATION = self.run_command("pipenv --venv").out + def cleanup(self): + log.info("Removing project's virtual emvironment %s", self.VENV_LOCATION) + self.remove_venv() + log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) + + def to_cache_path(self, project_path): + """ + Translates a path from the root of the project to the equivalent path in + the local cache. + + e.x.: '/a/b.py' -> 'python-cloned-projects-cache/project_name/a/b.py' + """ + # strip the leading '/' so that we can join it properly + file_path = os.path.relpath(project_path, "/") + + return os.path.join(self.CLONED_PROJECT_PATH, file_path) + + def ensure_venv_created(self): + ''' + This runs a noop pipenv command, which will + create the venv if it doesn't exist as a side effect. + ''' + self.run_command("true") + + def remove_venv(self): + self.run_command("pipenv --rm") + + def get_package_information(self): + project_packages = self.project_packages() + dependencies = self.external_dependencies() + + out = [] + for package in project_packages: + out.append({ + "package": { + "name": package + }, + "dependencies": dependencies + }) + return out + + def project_packages(self): + ''' + Provides a list of all packages declared in the project + ''' + script = [ + "import json", + "import setuptools", + "pkgs = setuptools.find_packages()", + "print(json.dumps(pkgs))" + ] + + c = self.run_command("python -c '{}'".format(";".join(script))) + return json.loads(c.out) + + def external_dependencies(self): + ''' + Provides a list of third party packages that the + project depends on. + ''' + deps = json.loads(self.run_command( + "pip list --local --format json").out) + out = [ + { + # TODO - is this ever not a dependency? + "attributes": { + "name": "cpython", + "repoURL": "git://github.com/python/cpython" + } + } + ] + + for dep in deps: + dep_name = dep["name"] + if dep_name not in set(["pip", "wheel"]): + out.append({"attributes": {"name": dep["name"]}}) + + return out + + def run_command(self, command, **kwargs): + ''' + Runs the given command inside the context + of the project: + + Context means: + - the command's cwd is the cached project directory + - the projects virtual environment is loaded into + the command's environment + ''' + kwargs["cwd"] = self.CLONED_PROJECT_PATH + + # add pipenv prefix + if type(command) is str: + command = "pipenv run {}".format(command) + else: + command = ["pipenv", "run"].append(command) + + return delegator.run(command, **kwargs) diff --git a/langserver/jedi.py b/langserver/jedi.py index b4fd874..242f716 100644 --- a/langserver/jedi.py +++ b/langserver/jedi.py @@ -1,34 +1,8 @@ -from os import path as filepath -import os import jedi import jedi._compatibility import opentracing -from typing import List - -from .fs import RemoteFileSystem, TestFileSystem - - -class Module: - def __init__(self, name, path, is_package=False): - self.name = name - self.path = path - self.is_package = is_package - - def __repr__(self): - return "PythonModule({}, {})".format(self.name, self.path) - - -class DummyFile: - def __init__(self, contents): - self.contents = contents - - def read(self): - return self.contents - - def close(self): - pass class RemoteJedi: @@ -54,125 +28,16 @@ def new_script(self, *args, **kwargs): def _new_script_impl(self, parent_span, *args, **kwargs): path = kwargs.get("path") - trace = False if 'trace' in kwargs: - trace = True del kwargs['trace'] - def find_module_remote(string, dir=None, fullname=None): - """A swap-in replacement for Jedi's find module function that uses - the remote fs to resolve module imports.""" - if dir is None: - dir = get_module_search_paths(string, path) - with opentracing.start_child_span( - parent_span, - "find_module_remote_callback") as find_module_span: - if trace: - print("find_module_remote", string, dir, fullname) - - the_module = None - - # TODO: move this bit of logic into the Workspace? - # default behavior is to search for built-ins first, but skip - # this if we're actually in the stdlib repo - if fullname and not self.workspace.is_stdlib: - the_module = self.workspace.find_stdlib_module(fullname) - - if the_module == "native": # break if we get a native module - raise ImportError( - 'Module "{}" not found in {}', string, dir) - - # TODO: use this clause's logic for the other clauses too - # (stdlib and external modules) after searching for built-ins, - # search the current project - if not the_module: - module_file, module_path, is_package = self.workspace.find_internal_module( - string, fullname, dir) - if module_file or module_path: - if is_package and module_path.endswith(".py"): - module_path = os.path.dirname(module_path) - return module_file, module_path, is_package + if self.workspace is not None: + path = self.workspace.to_cache_path(path) + venv_path = self.workspace.VENV_LOCATION - # finally, search 3rd party dependencies - if not the_module: - the_module = self.workspace.find_external_module(fullname) - - if not the_module: - raise ImportError( - 'Module "{}" not found in {}', string, dir) - - is_package = the_module.is_package - module_file = self.workspace.open_module_file( - the_module, find_module_span) - module_path = the_module.path - if is_package and the_module.is_namespace_package: - module_path = jedi._compatibility.ImplicitNSInfo( - fullname, [module_path]) - is_package = False - elif is_package and module_path.endswith(".py"): - module_path = filepath.dirname(module_path) - return module_file, module_path, is_package - - # TODO: update this to use the workspace's module indices - def list_modules() -> List[str]: - if trace: - print("list_modules") - modules = [ - f for f in self.fs.walk(self.root_path) - if f.lower().endswith(".py") - ] - return modules - - def load_source(path) -> str: - with opentracing.start_child_span( - parent_span, "load_source_callback") as load_source_span: - load_source_span.set_tag("path", path) - if trace: - print("load_source", path) - result = self.fs.open(path, load_source_span) - return result - - # TODO(keegan) It shouldn't matter if we are using a remote fs or not. - # Consider other ways to hook into the import system. - # TODO(aaron) Also, it shouldn't matter whether we're using a "real" - # filesystem or our test harness filesystem - if isinstance(self.fs, RemoteFileSystem) or isinstance( - self.fs, TestFileSystem): kwargs.update( - find_module=find_module_remote, - list_modules=list_modules, - load_source=load_source, - fs=self.fs + venv_path=venv_path, + path=path ) return jedi.api.Script(*args, **kwargs) - - -def get_module_search_paths(module_name, script_file_path): - """Provides an ordered list of directories in the workspace to search for - the given 'module_name', starting from the directory that the script is - operating on. - - This mimics Jedi's modifications of sys.path that it uses during module resolution. - See: - https://sourcegraph.com/github.com/sourcegraph/jedi/-/blob/jedi/evaluate/imports.py#L237:9 - - Note that this does not handle some corner cases, see: - https://www.python.org/dev/peps/pep-0235/ - """ - for parent in traverse_parents(script_file_path): - if os.path.basename(parent) == module_name: - yield parent - - -def traverse_parents(path): - ''' - Returns all parent directories for the given file path - Copied from: - https://sourcegraph.com/github.com/sourcegraph/jedi/-/blob/jedi/evaluate/sys_path.py#L228:5 - ''' - while True: - new = os.path.dirname(path) - if new == path: - return - path = new - yield path diff --git a/langserver/langserver.py b/langserver/langserver.py index 61edbab..dfd8e64 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -15,6 +15,7 @@ from .symbols import extract_symbols, workspace_symbols from .definitions import targeted_symbol from .references import get_references +from .clone_workspace import CloneWorkspace log = logging.getLogger(__name__) @@ -197,8 +198,8 @@ def test_initialize(self, request, fs): self.fs = fs self.streaming = False - self.workspace = Workspace(self.fs, self.root_path, - params["originalRootPath"]) + self.workspace = CloneWorkspace(self.fs, self.root_path, + params["originalRootPath"]) return { "capabilities": { @@ -658,7 +659,7 @@ def serve_document_symbols(self, request): return [s.json_object() for s in extract_symbols(source, path)] def serve_x_packages(self, request): - return self.workspace.get_package_information(request["span"]) + return self.workspace.get_package_information() def serve_x_dependencies(self, request): return self.workspace.get_dependencies(request["span"]) diff --git a/test/test_clone_workspace.py b/test/test_clone_workspace.py index 8bd81c8..83a3487 100644 --- a/test/test_clone_workspace.py +++ b/test/test_clone_workspace.py @@ -15,7 +15,6 @@ def test_data(): class TestCloningWorkspace: def test_clone(self, test_data): workspace, repoPath = test_data - workspace.clone_project() c = delegator.run("diff -r {} {}".format(repoPath, workspace.CLONED_PROJECT_PATH)) diff --git a/test/test_fizzbuzz.py b/test/test_fizzbuzz.py index 6688794..206f459 100644 --- a/test/test_fizzbuzz.py +++ b/test/test_fizzbuzz.py @@ -1,531 +1,531 @@ from .harness import Harness +import pytest -fizzbuzz_workspace = Harness("repos/fizzbuzz_service") -fizzbuzz_workspace.initialize("") +@pytest.fixture() +def fizzbuzz_workspace(): + fizzbuzz_workspace = Harness("repos/fizzbuzz_service") + fizzbuzz_workspace.initialize("repos/fizzbuzz_service") + yield fizzbuzz_workspace + fizzbuzz_workspace.exit() -def test_x_packages(): - result = fizzbuzz_workspace.x_packages() - assert len(result) == 1 - package = result[0] +class TestFizzBuzzWorkspace: + def test_x_packages(self, fizzbuzz_workspace): + result = fizzbuzz_workspace.x_packages() - assert "package" in package - assert package["package"] == {'name': 'fizzbuzz_service'} + assert len(result) == 7 + package = None + for package in result: + if "package" in package: + if package["package"] == {'name': 'fizzbuzz_service'}: + break - assert "dependencies" in package + assert "package" in package + assert package["package"] == {'name': 'fizzbuzz_service'} - dependencies = {d["attributes"]["name"] for d in package["dependencies"]} + assert "dependencies" in package - assert dependencies == { - "setuptools", - "cpython" - } - - -def test_local_hover(): - uri = "file:///fizzbuzz_service/loopers/number_looper.py" - line, col = 2, 7 - result = fizzbuzz_workspace.hover(uri, line, col) - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'class NumberLooper(param start, param end)' - }, - 'Very important class that is capable of gathering all the number strings ' - 'in [start, end)' - ] - } + dependencies = {d["attributes"]["name"] + for d in package["dependencies"]} + assert dependencies == { + "setuptools", + "cpython" + } -def test_local_package_cross_module_hover(): - uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - line, col = 4, 22 - result = fizzbuzz_workspace.hover(uri, line, col) + def test_local_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/loopers/number_looper.py" + line, col = 2, 7 + result = fizzbuzz_workspace.hover(uri, line, col) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'class NumberLooper(param start, param end)' + }, + 'Very important class that is capable of gathering all the number strings ' + 'in [start, end)' + ] + } - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'def decide_output_for_number(param number)' - }, - 'Decides the output for a given number' - ] - } + def test_local_package_cross_module_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + line, col = 4, 22 + result = fizzbuzz_workspace.hover(uri, line, col) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'def decide_output_for_number(param number)' + }, + 'Decides the output for a given number' + ] + } -def test_cross_package_hover(): - uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" - line, col = 5, 31 - result = fizzbuzz_workspace.hover(uri, line, col) - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'def should_fizz(param number)' - }, - 'Whether or not "fizz" should be printed for this number' - ] - } + def test_cross_package_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" + line, col = 5, 31 + result = fizzbuzz_workspace.hover(uri, line, col) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'def should_fizz(param number)' + }, + 'Whether or not "fizz" should be printed for this number' + ] + } + def test_std_lib_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/__main__.py" + line, col = 5, 10 + result = fizzbuzz_workspace.hover(uri, line, col) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'def print(param value, param ..., param sep, param ' + 'end, param file, param flush)' + }, + "print(value, ..., sep=' ', end='\\n', file=sys.stdout, " + 'flush=False)\n' + '\n' + 'Prints the values to a stream, or to sys.stdout by default.\n' + 'Optional keyword arguments:\n' + 'file: a file-like object (stream); defaults to the current ' + 'sys.stdout.\n' + 'sep: string inserted between values, default a space.\n' + 'end: string appended after the last value, default a newline.\n' + 'flush: whether to forcibly flush the stream.' + ] + } -def test_std_lib_hover(): - uri = "file:///fizzbuzz_service/__main__.py" - line, col = 5, 10 - result = fizzbuzz_workspace.hover(uri, line, col) - assert result == { - 'contents': [ + def test_local_defintion(self, fizzbuzz_workspace): + uri = "/fizzbuzz_service/string_deciders/number_decision.py" + line, col = 21, 21 + result = fizzbuzz_workspace.definition(uri, line, col) + assert result == [ { - 'language': 'python', - 'value': 'def print(param value, param ..., param sep, param ' - 'end, param file, param flush)' - }, - "print(value, ..., sep=' ', end='\\n', file=sys.stdout, " - 'flush=False)\n' - '\n' - 'Prints the values to a stream, or to sys.stdout by default.\n' - 'Optional keyword arguments:\n' - 'file: a file-like object (stream); defaults to the current ' - 'sys.stdout.\n' - 'sep: string inserted between values, default a space.\n' - 'end: string appended after the last value, default a newline.\n' - 'flush: whether to forcibly flush the stream.' + 'symbol': { + 'package': { + 'name': 'fizzbuzz_service' + }, + 'name': 'OutputDecision', + 'container': 'fizzbuzz_service.string_deciders.number_decision', + 'kind': 'class', + 'file': 'number_decision.py', + 'position': { + 'line': 5, + 'character': 6 + } + }, + 'location': { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 5, + 'character': 6 + }, + 'end': { + 'line': 5, + 'character': 20 + } + } + } + } ] - } - -def test_local_defintion(): - uri = "/fizzbuzz_service/string_deciders/number_decision.py" - line, col = 21, 21 - result = fizzbuzz_workspace.definition(uri, line, col) - assert result == [ - { + def test_local_package_cross_module_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + line, col = 4, 25 + result = fizzbuzz_workspace.definition(uri, line, col) + definition = { 'symbol': { 'package': { 'name': 'fizzbuzz_service' }, - 'name': 'OutputDecision', + 'name': 'decide_output_for_number', 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'class', + 'kind': 'def', 'file': 'number_decision.py', 'position': { - 'line': 5, - 'character': 6 + 'line': 12, + 'character': 4 } }, 'location': { 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', 'range': { 'start': { - 'line': 5, - 'character': 6 + 'line': 12, + 'character': 4 }, 'end': { - 'line': 5, - 'character': 20 + 'line': 12, + 'character': 28 } } } } - ] - -def test_local_package_cross_module_definition(): - uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - line, col = 4, 25 - result = fizzbuzz_workspace.definition(uri, line, col) - definition = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'decide_output_for_number', - 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'def', - 'file': 'number_decision.py', - 'position': { - 'line': 12, - 'character': 4 - } - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 12, - 'character': 4 + # from the import statement at the top of the file + assignment = { + 'symbol': { + 'package': { + 'name': 'fizzbuzz_service' }, - 'end': { - 'line': 12, - 'character': 28 - } - } - } - } - - # from the import statement at the top of the file - assignment = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'decide_output_for_number', - 'container': 'fizzbuzz_service.string_deciders.number_decider', - 'kind': 'def', - 'file': 'number_decider.py', - 'position': { - 'line': 0, - 'character': 45 - } - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { + 'name': 'decide_output_for_number', + 'container': 'fizzbuzz_service.string_deciders.number_decider', + 'kind': 'def', + 'file': 'number_decider.py', + 'position': { 'line': 0, 'character': 45 - }, - 'end': { - 'line': 0, - 'character': 69 + } + }, + 'location': { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + 'range': { + 'start': { + 'line': 0, + 'character': 45 + }, + 'end': { + 'line': 0, + 'character': 69 + } } } } - } - - assert len(result) == 2 - assert definition in result - assert assignment in result + assert len(result) == 2 + assert definition in result + assert assignment in result -def test_cross_package_definition(): - uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" - line, col = 5, 57 - result = fizzbuzz_workspace.definition(uri, line, col) + def test_cross_package_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" + line, col = 5, 57 + result = fizzbuzz_workspace.definition(uri, line, col) - definition = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'should_buzz', - 'container': 'fizzbuzz_service.checkers.buzz.buzz_checker', - 'kind': 'def', - 'file': 'buzz_checker.py', - 'position': { - 'line': 0, - 'character': 4 - } - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/checkers/buzz/buzz_checker.py', - 'range': { - 'start': { - 'line': 0, - 'character': 4 + definition = { + 'symbol': { + 'package': { + 'name': 'fizzbuzz_service' }, - 'end': { + 'name': 'should_buzz', + 'container': 'fizzbuzz_service.checkers.buzz.buzz_checker', + 'kind': 'def', + 'file': 'buzz_checker.py', + 'position': { 'line': 0, - 'character': 15 + 'character': 4 + } + }, + 'location': { + 'uri': 'file:///fizzbuzz_service/checkers/buzz/buzz_checker.py', + 'range': { + 'start': { + 'line': 0, + 'character': 4 + }, + 'end': { + 'line': 0, + 'character': 15 + } } } } - } - # from the import statement at the top of the file - assignment = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'should_buzz', - 'container': 'fizzbuzz_service.checkers.fizzbuzz.fizzbuzz_checker', - 'kind': 'def', - 'file': 'fizzbuzz_checker.py', - 'position': { - 'line': 1, - 'character': 32 - }, - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py', - 'range': { - 'start': { + # from the import statement at the top of the file + assignment = { + 'symbol': { + 'package': { + 'name': 'fizzbuzz_service' + }, + 'name': 'should_buzz', + 'container': 'fizzbuzz_service.checkers.fizzbuzz.fizzbuzz_checker', + 'kind': 'def', + 'file': 'fizzbuzz_checker.py', + 'position': { 'line': 1, 'character': 32 }, - 'end': { - 'line': 1, - 'character': 43 + }, + 'location': { + 'uri': 'file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py', + 'range': { + 'start': { + 'line': 1, + 'character': 32 + }, + 'end': { + 'line': 1, + 'character': 43 + } } } } - } - - assert len(result) == 2 - - assert definition in result - assert assignment in result - - -def test_local_package_cross_module_import_definition(): - uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - line, col = 0, 14 - result = fizzbuzz_workspace.definition(uri, line, col) - - assert len(result) == 1 - definition = result[0] - - assert "symbol" in definition - assert definition["symbol"] == { - 'package': { - 'name': 'fizzbuzz_service'}, - 'name': 'number_decision', - 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'module', - 'file': 'number_decision.py', - 'position': { - 'line': 0, - 'character': 0}, - } - - assert "location" in definition - location = definition["location"] - assert "uri" in location - assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/number_decision.py' - - # TODO: In the case of a module, does the range have any meaning? - - -def test_cross_package_import_definition(): - uri = "file:///fizzbuzz_service/loopers/number_looper.py" - line, col = 0, 31 - result = fizzbuzz_workspace.definition(uri, line, col) - - assert len(result) == 1 - definition = result[0] - - assert "symbol" in definition - assert definition["symbol"] == { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'string_deciders', - 'container': 'fizzbuzz_service.string_deciders', - 'kind': 'module', - 'file': '__init__.py', - 'position': { - 'line': 0, - 'character': 0 - }, - } - - assert "location" in definition - location = definition["location"] - assert "uri" in location - assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/__init__.py' - - # TODO: In the case of a module, does the range have any meaning? - - -def test_std_lib_definition(): - uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' - line, col = 5, 23 - result = fizzbuzz_workspace.definition(uri, line, col) - - definition = { - 'symbol': { + + assert len(result) == 2 + + assert definition in result + assert assignment in result + + def test_local_package_cross_module_import_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + line, col = 0, 14 + result = fizzbuzz_workspace.definition(uri, line, col) + + assert len(result) == 1 + definition = result[0] + + assert "symbol" in definition + assert definition["symbol"] == { 'package': { - 'name': 'cpython' - }, - 'name': 'Enum', - 'container': 'enum', - 'kind': 'class', - 'path': 'Lib/enum.py', - 'file': 'enum.py', + 'name': 'fizzbuzz_service'}, + 'name': 'number_decision', + 'container': 'fizzbuzz_service.string_deciders.number_decision', + 'kind': 'module', + 'file': 'number_decision.py', 'position': { - 'line': 508, - 'character': 6 - }, - }, - 'location': None - } + 'line': 0, + 'character': 0}, + } - # from the import statement at the top of the file - assignment = { - 'symbol': { + assert "location" in definition + location = definition["location"] + assert "uri" in location + assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/number_decision.py' + + # TODO: In the case of a module, does the range have any meaning? + + def test_cross_package_import_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/loopers/number_looper.py" + line, col = 0, 31 + result = fizzbuzz_workspace.definition(uri, line, col) + + assert len(result) == 1 + definition = result[0] + + assert "symbol" in definition + assert definition["symbol"] == { 'package': { 'name': 'fizzbuzz_service' }, - 'name': 'Enum', - 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'class', - 'file': 'number_decision.py', + 'name': 'string_deciders', + 'container': 'fizzbuzz_service.string_deciders', + 'kind': 'module', + 'file': '__init__.py', 'position': { 'line': 0, - 'character': 17 + 'character': 0 }, - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { + } + + assert "location" in definition + location = definition["location"] + assert "uri" in location + assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/__init__.py' + + # TODO: In the case of a module, does the range have any meaning? + + def test_std_lib_definition(self, fizzbuzz_workspace): + uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' + line, col = 5, 23 + result = fizzbuzz_workspace.definition(uri, line, col) + + definition = { + 'symbol': { + 'package': { + 'name': 'cpython' + }, + 'name': 'Enum', + 'container': 'enum', + 'kind': 'class', + 'path': 'Lib/enum.py', + 'file': 'enum.py', + 'position': { + 'line': 508, + 'character': 6 + }, + }, + 'location': None + } + + # from the import statement at the top of the file + assignment = { + 'symbol': { + 'package': { + 'name': 'fizzbuzz_service' + }, + 'name': 'Enum', + 'container': 'fizzbuzz_service.string_deciders.number_decision', + 'kind': 'class', + 'file': 'number_decision.py', + 'position': { 'line': 0, 'character': 17 }, - 'end': { - 'line': 0, - 'character': 21 + }, + 'location': { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 0, + 'character': 17 + }, + 'end': { + 'line': 0, + 'character': 21 + } } } } - } - - assert len(result) == 2 - assert definition in result - assert assignment in result + assert len(result) == 2 + assert definition in result + assert assignment in result -def test_local_references(): - uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' - line, col = 5, 13 - result = fizzbuzz_workspace.references(uri, line, col) + def test_local_references(self, fizzbuzz_workspace): + uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' + line, col = 5, 13 + result = fizzbuzz_workspace.references(uri, line, col) - references = [ - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { - 'line': 6, - 'character': 19 - }, - 'end': { - 'line': 6, - 'character': 33 + references = [ + { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + 'range': { + 'start': { + 'line': 6, + 'character': 19 + }, + 'end': { + 'line': 6, + 'character': 33 + } } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { - 'line': 9, - 'character': 19 - }, - 'end': { - 'line': 9, - 'character': 33 + }, + { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + 'range': { + 'start': { + 'line': 9, + 'character': 19 + }, + 'end': { + 'line': 9, + 'character': 33 + } } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { - 'line': 12, - 'character': 19 - }, - 'end': { - 'line': 12, - 'character': 33 + }, + { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + 'range': { + 'start': { + 'line': 12, + 'character': 19 + }, + 'end': { + 'line': 12, + 'character': 33 + } } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 15, - 'character': 15 - }, - 'end': { - 'line': 15, - 'character': 29 + }, + { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 15, + 'character': 15 + }, + 'end': { + 'line': 15, + 'character': 29 + } } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 18, - 'character': 15 - }, - 'end': { - 'line': 18, - 'character': 29 + }, + { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 18, + 'character': 15 + }, + 'end': { + 'line': 18, + 'character': 29 + } } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 21, - 'character': 15 - }, - 'end': - { - 'line': 21, - 'character': 29 + }, + { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 21, + 'character': 15 + }, + 'end': + { + 'line': 21, + 'character': 29 + } } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 23, - 'character': 11 - }, - 'end': { - 'line': 23, - 'character': 25 + }, + { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 23, + 'character': 11 + }, + 'end': { + 'line': 23, + 'character': 25 + } } - } - }, - ] + }, + ] + + assert len(references) == len(result) + for ref in references: + assert ref in result - assert len(references) == len(result) - for ref in references: - assert ref in result + def test_x_references(self, fizzbuzz_workspace): + result = fizzbuzz_workspace.x_references("enum", "Enum") + import_ref = { + 'reference': { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 0, + 'character': 0}, + 'end': { + 'line': 0, + 'character': 4}}}, + 'symbol': { + 'container': 'enum', + 'name': 'Enum'}} -def test_x_references(): - result = fizzbuzz_workspace.x_references("enum", "Enum") + base_class_ref = { + 'reference': { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 5, + 'character': 21}, + 'end': { + 'line': 5, + 'character': 25}}}, + 'symbol': { + 'container': 'enum', + 'name': 'Enum'}} - import_ref = { - 'reference': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 0, - 'character': 0}, - 'end': { - 'line': 0, - 'character': 4}}}, - 'symbol': { - 'container': 'enum', - 'name': 'Enum'}} - - base_class_ref = { - 'reference': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 5, - 'character': 21}, - 'end': { - 'line': 5, - 'character': 25}}}, - 'symbol': { - 'container': 'enum', - 'name': 'Enum'}} - - assert len(result) == 2 - - assert import_ref in result - assert base_class_ref in result + assert len(result) == 2 + + assert import_ref in result + assert base_class_ref in result From 270f4cf4be4a4286a7a3f5a1978d867aaed24fc5 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 1 Mar 2018 17:15:31 -0800 Subject: [PATCH 04/45] make fizzbuzz refs test pass --- langserver/clone_workspace.py | 16 ++++++++++++++-- langserver/langserver.py | 7 ++++--- test/test_clone_workspace.py | 1 - 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index fe275db..48060dc 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -37,8 +37,8 @@ def __init__(self, fs: FileSystem, project_root: str, # TODO: allow different Python versions per project/workspace self.PYTHON_PATH = GlobalConfig.PYTHON_PATH - self.CLONED_PROJECT_PATH = os.path.join( - GlobalConfig.CLONED_PROJECT_PATH, self.key) + self.CLONED_PROJECT_PATH = os.path.abspath(os.path.join( + GlobalConfig.CLONED_PROJECT_PATH, self.key)) log.debug("Setting Python path to %s", self.PYTHON_PATH) log.debug("Setting Cloned Project path to %s", self.CLONED_PROJECT_PATH) @@ -80,6 +80,18 @@ def to_cache_path(self, project_path): return os.path.join(self.CLONED_PROJECT_PATH, file_path) + def from_cache_path(self, cache_path): + """ + Translates a path in the cache to the equivalent path in + the project. + + e.x.: 'python-cloned-projects-cache/project_name/a/b.py' -> '/a/b.py' + """ + file_path = os.path.relpath(cache_path, self.CLONED_PROJECT_PATH) + if not file_path.startswith("/"): + file_path = "/" + file_path + return file_path + def ensure_venv_created(self): ''' This runs a noop pipenv command, which will diff --git a/langserver/langserver.py b/langserver/langserver.py index dfd8e64..8ffea4f 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -522,13 +522,14 @@ def serve_references(self, request): self.conn.send_notification( "$/partialResult", partial_initializer) if self.streaming else None json_patch = [] - package_cache_path = os.path.abspath(self.workspace.PACKAGES_PATH) + # package_cache_path = os.path.abspath(self.workspace.PACKAGES_PATH) for u in usages: + u.module_path = self.workspace.from_cache_path(u.module_path) if u.is_definition(): continue # filter out any results from files that are cached on the local fs - if u.module_path.startswith(package_cache_path): - continue + # if u.module_path.startswith(package_cache_path): + # continue if u.module_path.startswith(self.workspace.PYTHON_PATH): continue location = { diff --git a/test/test_clone_workspace.py b/test/test_clone_workspace.py index 83a3487..278b779 100644 --- a/test/test_clone_workspace.py +++ b/test/test_clone_workspace.py @@ -9,7 +9,6 @@ def test_data(): repoPath = "repos/fizzbuzz_service" workspace = CloneWorkspace(TestFileSystem(repoPath), repoPath, repoPath) yield (workspace, repoPath) - workspace.cleanup() class TestCloningWorkspace: From 1459573f1897967afe1e9636cb3207e407515774 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Fri, 2 Mar 2018 19:37:07 -0800 Subject: [PATCH 05/45] wip --- Makefile | 2 +- Pipfile.lock | 5 - langserver/clone_workspace.py | 78 ++- langserver/jedi.py | 2 +- langserver/langserver.py | 96 ++-- test/.cache/v/cache/lastfailed | 46 +- test/test_fizzbuzz.py | 920 +++++++++++++++++---------------- 7 files changed, 615 insertions(+), 534 deletions(-) diff --git a/Makefile b/Makefile index 508cedb..5cf6f4d 100644 --- a/Makefile +++ b/Makefile @@ -9,4 +9,4 @@ lint: test: # pipenv run pytest test_langserver.py - cd ./test && pipenv run pytest test_fizzbuzz.py + cd ./test && pipenv run pytest test_fizzbuzz.py -vv diff --git a/Pipfile.lock b/Pipfile.lock index 1b54c75..7f1ef47 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -139,11 +139,6 @@ "git": "git://github.com/sourcegraph/requirements-parser.git", "ref": "69f1a9cb916b2995843c3ea9b988da46c9dd65c7" }, - "requirements-parser": { - "editable": true, - "git": "git://github.com/sourcegraph/requirements-parser.git", - "ref": "69f1a9cb916b2995843c3ea9b988da46c9dd65c7" - }, "six": { "hashes": [ "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 48060dc..43c7c7b 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -6,6 +6,9 @@ from shutil import rmtree import delegator import json +from functools import lru_cache +from enum import Enum +import pathlib log = logging.getLogger(__name__) @@ -15,6 +18,7 @@ class CloneWorkspace: def __init__(self, fs: FileSystem, project_root: str, original_root_path: str= ""): + self.fs = fs self.PROJECT_ROOT = project_root self.repo = None self.hash = None @@ -35,40 +39,39 @@ def __init__(self, fs: FileSystem, project_root: str, if self.hash: self.key = ".".join((self.key, self.hash)) - # TODO: allow different Python versions per project/workspace - self.PYTHON_PATH = GlobalConfig.PYTHON_PATH + self.PYTHON_PATH = os.path.abspath(GlobalConfig.PYTHON_PATH) self.CLONED_PROJECT_PATH = os.path.abspath(os.path.join( GlobalConfig.CLONED_PROJECT_PATH, self.key)) + log.debug("Setting Python path to %s", self.PYTHON_PATH) log.debug("Setting Cloned Project path to %s", self.CLONED_PROJECT_PATH) - self.fs = fs - # Clone the project from the provided filesystem into the local # cache all_files = self.fs.walk(self.PROJECT_ROOT) for file_path in all_files: - cache_file_path = self.to_cache_path(file_path) - if os.path.exists(cache_file_path): - continue + cache_file_path = self.project_to_cache_path(file_path) os.makedirs(os.path.dirname(cache_file_path), exist_ok=True) file_contents = self.fs.open(file_path) with open(cache_file_path, "w") as f: f.write(file_contents) + @property + @lru_cache() + def VENV_LOCATION(self): self.ensure_venv_created() - self.VENV_LOCATION = self.run_command("pipenv --venv").out + return self.run_command("pipenv --venv").out.rstrip() def cleanup(self): - log.info("Removing project's virtual emvironment %s", self.VENV_LOCATION) + log.info("Removing project's virtual environment %s", self.VENV_LOCATION) self.remove_venv() log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) - def to_cache_path(self, project_path): + def project_to_cache_path(self, project_path): """ Translates a path from the root of the project to the equivalent path in the local cache. @@ -80,7 +83,7 @@ def to_cache_path(self, project_path): return os.path.join(self.CLONED_PROJECT_PATH, file_path) - def from_cache_path(self, cache_path): + def cache_path_to_project_path(self, cache_path): """ Translates a path in the cache to the equivalent path in the project. @@ -100,7 +103,37 @@ def ensure_venv_created(self): self.run_command("true") def remove_venv(self): - self.run_command("pipenv --rm") + self.run_command("pipenv --rm", no_prefix=True) + + def get_module_info(self, raw_module_path): + module_path = pathlib.Path(raw_module_path) + + import pdb + pdb.set_trace() + + sys_std_lib_path = pathlib.Path(self.PYTHON_PATH) + if sys_std_lib_path in module_path.parents: + return (ModuleKind.STANDARD_LIBRARY, path.relative_to(sys_std_lib_path)) + + venv_path = pathlib.Path(self.VENV_LOCATION) / "lib" + if venv_path in module_path.parents: + # The python libraries in a venv are stored under + # VENV_LOCATION/lib/(some_python_version) + + python_version = module_path.relative_to(venv_path).parts[0] + + venv_lib_path = venv_path / python_version + venv_ext_packages_path = venv_lib_path / "site-packages" + + if venv_ext_packages_path in module_path.parents: + return (ModuleKind.EXTERNAL_DEPENDENCY, module_path.relative_to(venv_ext_packages_path)) + return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(venv_lib_path)) + + project_path = pathlib.Path(self.CLONED_PROJECT_PATH) + if project_path in module_path.parents: + return (ModuleKind.PROJECT, module_path.relative_to(project_path)) + + return (ModuleKind.UNKNOWN, module_path) def get_package_information(self): project_packages = self.project_packages() @@ -116,6 +149,7 @@ def get_package_information(self): }) return out + @lru_cache() def project_packages(self): ''' Provides a list of all packages declared in the project @@ -150,11 +184,11 @@ def external_dependencies(self): for dep in deps: dep_name = dep["name"] if dep_name not in set(["pip", "wheel"]): - out.append({"attributes": {"name": dep["name"]}}) + out.append({"attributes": {"name": dep_name}}) return out - def run_command(self, command, **kwargs): + def run_command(self, command, no_prefix=False, **kwargs): ''' Runs the given command inside the context of the project: @@ -167,9 +201,17 @@ def run_command(self, command, **kwargs): kwargs["cwd"] = self.CLONED_PROJECT_PATH # add pipenv prefix - if type(command) is str: - command = "pipenv run {}".format(command) - else: - command = ["pipenv", "run"].append(command) + if not no_prefix: + if type(command) is str: + command = "pipenv run {}".format(command) + else: + command = ["pipenv", "run"].append(command) return delegator.run(command, **kwargs) + + +class ModuleKind(Enum): + PROJECT = 1 + STANDARD_LIBRARY = 2 + EXTERNAL_DEPENDENCY = 3 + UNKNOWN = 4 diff --git a/langserver/jedi.py b/langserver/jedi.py index 242f716..c572588 100644 --- a/langserver/jedi.py +++ b/langserver/jedi.py @@ -32,7 +32,7 @@ def _new_script_impl(self, parent_span, *args, **kwargs): del kwargs['trace'] if self.workspace is not None: - path = self.workspace.to_cache_path(path) + path = self.workspace.project_to_cache_path(path) venv_path = self.workspace.VENV_LOCATION kwargs.update( diff --git a/langserver/langserver.py b/langserver/langserver.py index 8ffea4f..0007ff5 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -15,7 +15,7 @@ from .symbols import extract_symbols, workspace_symbols from .definitions import targeted_symbol from .references import get_references -from .clone_workspace import CloneWorkspace +from .clone_workspace import CloneWorkspace, ModuleKind log = logging.getLogger(__name__) @@ -354,63 +354,61 @@ def serve_x_definition(self, request): return results for d in defs: - defining_module_path = d.module_path - defining_module = self.workspace.get_module_by_path( - defining_module_path) + kind, module_path = ModuleKind.UNKNOWN, "" + if d.module_path: + kind, module_path = self.workspace.get_module_info( + d.module_path) - if not defining_module and (not d.is_definition() or - d.line is None or d.column is None): + if (not d.is_definition() or + d.line is None or d.column is None): continue symbol_locator = {"symbol": None, "location": None} - if defining_module and not defining_module.is_stdlib: - # the module path doesn't map onto the repository structure - # because we're not fully installing - # dependency packages, so don't include it in the symbol - # descriptor - filename = os.path.basename(defining_module_path) - symbol_name = "" - symbol_kind = "" - if d.description: - symbol_name, symbol_kind = LangServer.name_and_kind( - d.description) - symbol_locator["symbol"] = { - "package": { - "name": defining_module.qualified_name.split(".")[0], - }, - "name": symbol_name, - "container": defining_module.qualified_name, - "kind": symbol_kind, - "file": filename - } - - elif defining_module and defining_module.is_stdlib: - rel_path = os.path.relpath(defining_module_path, - self.workspace.PYTHON_PATH) - filename = os.path.basename(defining_module_path) - symbol_name = "" - symbol_kind = "" - if d.description: - symbol_name, symbol_kind = LangServer.name_and_kind( - d.description) - symbol_locator["symbol"] = { - "package": { - "name": "cpython", - }, - "name": symbol_name, - "container": defining_module.qualified_name, - "kind": symbol_kind, - "path": os.path.join(GlobalConfig.STDLIB_SRC_PATH, - rel_path), - "file": filename - } + if kind is not ModuleKind.UNKNOWN: + if kind == ModuleKind.STANDARD_LIBRARY: + filename = module_path.name + symbol_name = "" + symbol_kind = "" + if d.description: + symbol_name, symbol_kind = LangServer.name_and_kind( + d.description) + symbol_locator["symbol"] = { + "package": { + "name": "cpython", + }, + "name": symbol_name, + "container": d.full_name, + "kind": symbol_kind, + "path": str(GlobalConfig.STDLIB_SRC_PATH / module_path), + "file": filename + } + else: + # the module path doesn't map onto the repository structure + # because we're not fully installing + # dependency packages, so don't include it in the symbol + # descriptor + filename = module_path.name + symbol_name = "" + symbol_kind = "" + if d.description: + symbol_name, symbol_kind = LangServer.name_and_kind( + d.description) + symbol_locator["symbol"] = { + "package": { + "name": d.full_name.split(".")[0], + }, + "name": symbol_name, + "container": d.full_name, + "kind": symbol_kind, + "file": filename + } if (d.is_definition() and d.line is not None and d.column is not None): location = { # TODO(renfred) determine why d.module_path is empty. - "uri": "file://" + (d.module_path or path), + "uri": "file://" + (str(module_path) or path), "range": { "start": { "line": d.line - 1, @@ -429,7 +427,7 @@ def serve_x_definition(self, request): "start"] # set the full location if the definition is in this workspace - if not defining_module or not defining_module.is_external: + if not kind in [ModuleKind.UNKNOWN, ModuleKind.EXTERNAL_DEPENDENCY]: symbol_locator["location"] = location results.append(symbol_locator) diff --git a/test/.cache/v/cache/lastfailed b/test/.cache/v/cache/lastfailed index 9e26dfe..2f3efc7 100644 --- a/test/.cache/v/cache/lastfailed +++ b/test/.cache/v/cache/lastfailed @@ -1 +1,45 @@ -{} \ No newline at end of file +{ + "test_clone_workspace.py::TestCloningWorkspace::()::test_clone": true, + "test_conflictingdeps.py::TestConflictingDependencies::()::test_hover_packages_conflicting_module_names": true, + "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data0]": true, + "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data1]": true, + "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data2]": true, + "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data3]": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_cross_package_definition": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_cross_package_import_definition": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_defintion": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_package_cross_module_definition": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_package_cross_module_import_definition": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_references": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_std_lib_definition": true, + "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_x_references": true, + "test_fizzbuzz.py::test_cross_package_definition": true, + "test_fizzbuzz.py::test_cross_package_import_definition": true, + "test_fizzbuzz.py::test_local_defintion": true, + "test_fizzbuzz.py::test_local_package_cross_module_definition": true, + "test_fizzbuzz.py::test_local_package_cross_module_import_definition": true, + "test_fizzbuzz.py::test_local_references": true, + "test_fizzbuzz.py::test_std_lib_definition": true, + "test_fizzbuzz.py::test_x_packages": true, + "test_flask.py::test_cross_module_definition": true, + "test_flask.py::test_cross_module_hover": true, + "test_flask.py::test_cross_package_definition": true, + "test_flask.py::test_cross_package_hover": true, + "test_flask.py::test_cross_repo_definition": true, + "test_flask.py::test_cross_repo_hover": true, + "test_flask.py::test_cross_repo_import_definition": true, + "test_flask.py::test_definition_of_definition": true, + "test_flask.py::test_local_definition": true, + "test_flask.py::test_local_package_import_definition": true, + "test_flask.py::test_local_references": true, + "test_flask.py::test_stdlib_definition": true, + "test_flask.py::test_x_packages": true, + "test_global_variables.py::test_name_definition": true, + "test_graphql_core.py::test_relative_import_definition": true, + "test_jedi.py::test_absolute_import_definiton": true, + "test_jedi.py::test_x_packages": true, + "test_modpkgsamename.py::TestExtPkgHasModuleWithSameName::()::test_hover_pkg_module_same_name": true, + "test_tensorflow_models.py::test_ad_hoc_module_definition": true, + "test_tensorflow_models.py::test_namespace_package_definition": true, + "test_thefuck.py::test_local_references": true +} \ No newline at end of file diff --git a/test/test_fizzbuzz.py b/test/test_fizzbuzz.py index 206f459..eea66c1 100644 --- a/test/test_fizzbuzz.py +++ b/test/test_fizzbuzz.py @@ -11,326 +11,326 @@ def fizzbuzz_workspace(): class TestFizzBuzzWorkspace: - def test_x_packages(self, fizzbuzz_workspace): - result = fizzbuzz_workspace.x_packages() - - assert len(result) == 7 - package = None - for package in result: - if "package" in package: - if package["package"] == {'name': 'fizzbuzz_service'}: - break - - assert "package" in package - assert package["package"] == {'name': 'fizzbuzz_service'} - - assert "dependencies" in package - - dependencies = {d["attributes"]["name"] - for d in package["dependencies"]} - - assert dependencies == { - "setuptools", - "cpython" - } - - def test_local_hover(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/loopers/number_looper.py" - line, col = 2, 7 - result = fizzbuzz_workspace.hover(uri, line, col) - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'class NumberLooper(param start, param end)' - }, - 'Very important class that is capable of gathering all the number strings ' - 'in [start, end)' - ] - } - - def test_local_package_cross_module_hover(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - line, col = 4, 22 - result = fizzbuzz_workspace.hover(uri, line, col) - - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'def decide_output_for_number(param number)' - }, - 'Decides the output for a given number' - ] - } - - def test_cross_package_hover(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" - line, col = 5, 31 - result = fizzbuzz_workspace.hover(uri, line, col) - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'def should_fizz(param number)' - }, - 'Whether or not "fizz" should be printed for this number' - ] - } - - def test_std_lib_hover(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/__main__.py" - line, col = 5, 10 - result = fizzbuzz_workspace.hover(uri, line, col) - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'def print(param value, param ..., param sep, param ' - 'end, param file, param flush)' - }, - "print(value, ..., sep=' ', end='\\n', file=sys.stdout, " - 'flush=False)\n' - '\n' - 'Prints the values to a stream, or to sys.stdout by default.\n' - 'Optional keyword arguments:\n' - 'file: a file-like object (stream); defaults to the current ' - 'sys.stdout.\n' - 'sep: string inserted between values, default a space.\n' - 'end: string appended after the last value, default a newline.\n' - 'flush: whether to forcibly flush the stream.' - ] - } - - def test_local_defintion(self, fizzbuzz_workspace): - uri = "/fizzbuzz_service/string_deciders/number_decision.py" - line, col = 21, 21 - result = fizzbuzz_workspace.definition(uri, line, col) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'OutputDecision', - 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'class', - 'file': 'number_decision.py', - 'position': { - 'line': 5, - 'character': 6 - } - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 5, - 'character': 6 - }, - 'end': { - 'line': 5, - 'character': 20 - } - } - } - } - ] - - def test_local_package_cross_module_definition(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - line, col = 4, 25 - result = fizzbuzz_workspace.definition(uri, line, col) - definition = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'decide_output_for_number', - 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'def', - 'file': 'number_decision.py', - 'position': { - 'line': 12, - 'character': 4 - } - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 12, - 'character': 4 - }, - 'end': { - 'line': 12, - 'character': 28 - } - } - } - } - - # from the import statement at the top of the file - assignment = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'decide_output_for_number', - 'container': 'fizzbuzz_service.string_deciders.number_decider', - 'kind': 'def', - 'file': 'number_decider.py', - 'position': { - 'line': 0, - 'character': 45 - } - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { - 'line': 0, - 'character': 45 - }, - 'end': { - 'line': 0, - 'character': 69 - } - } - } - } - - assert len(result) == 2 - - assert definition in result - assert assignment in result - - def test_cross_package_definition(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" - line, col = 5, 57 - result = fizzbuzz_workspace.definition(uri, line, col) - - definition = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'should_buzz', - 'container': 'fizzbuzz_service.checkers.buzz.buzz_checker', - 'kind': 'def', - 'file': 'buzz_checker.py', - 'position': { - 'line': 0, - 'character': 4 - } - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/checkers/buzz/buzz_checker.py', - 'range': { - 'start': { - 'line': 0, - 'character': 4 - }, - 'end': { - 'line': 0, - 'character': 15 - } - } - } - } - - # from the import statement at the top of the file - assignment = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'should_buzz', - 'container': 'fizzbuzz_service.checkers.fizzbuzz.fizzbuzz_checker', - 'kind': 'def', - 'file': 'fizzbuzz_checker.py', - 'position': { - 'line': 1, - 'character': 32 - }, - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py', - 'range': { - 'start': { - 'line': 1, - 'character': 32 - }, - 'end': { - 'line': 1, - 'character': 43 - } - } - } - } - - assert len(result) == 2 - - assert definition in result - assert assignment in result - - def test_local_package_cross_module_import_definition(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - line, col = 0, 14 - result = fizzbuzz_workspace.definition(uri, line, col) - - assert len(result) == 1 - definition = result[0] - - assert "symbol" in definition - assert definition["symbol"] == { - 'package': { - 'name': 'fizzbuzz_service'}, - 'name': 'number_decision', - 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'module', - 'file': 'number_decision.py', - 'position': { - 'line': 0, - 'character': 0}, - } - - assert "location" in definition - location = definition["location"] - assert "uri" in location - assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/number_decision.py' - - # TODO: In the case of a module, does the range have any meaning? - - def test_cross_package_import_definition(self, fizzbuzz_workspace): - uri = "file:///fizzbuzz_service/loopers/number_looper.py" - line, col = 0, 31 - result = fizzbuzz_workspace.definition(uri, line, col) - - assert len(result) == 1 - definition = result[0] - - assert "symbol" in definition - assert definition["symbol"] == { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'string_deciders', - 'container': 'fizzbuzz_service.string_deciders', - 'kind': 'module', - 'file': '__init__.py', - 'position': { - 'line': 0, - 'character': 0 - }, - } - - assert "location" in definition - location = definition["location"] - assert "uri" in location - assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/__init__.py' - - # TODO: In the case of a module, does the range have any meaning? + # def test_x_packages(self, fizzbuzz_workspace): + # result = fizzbuzz_workspace.x_packages() + + # assert len(result) == 7 + # package = None + # for package in result: + # if "package" in package: + # if package["package"] == {'name': 'fizzbuzz_service'}: + # break + + # assert "package" in package + # assert package["package"] == {'name': 'fizzbuzz_service'} + + # assert "dependencies" in package + + # dependencies = {d["attributes"]["name"] + # for d in package["dependencies"]} + + # assert dependencies == { + # "setuptools", + # "cpython" + # } + + # def test_local_hover(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/loopers/number_looper.py" + # line, col = 2, 7 + # result = fizzbuzz_workspace.hover(uri, line, col) + # assert result == { + # 'contents': [ + # { + # 'language': 'python', + # 'value': 'class NumberLooper(param start, param end)' + # }, + # 'Very important class that is capable of gathering all the number strings ' + # 'in [start, end)' + # ] + # } + + # def test_local_package_cross_module_hover(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + # line, col = 4, 22 + # result = fizzbuzz_workspace.hover(uri, line, col) + + # assert result == { + # 'contents': [ + # { + # 'language': 'python', + # 'value': 'def decide_output_for_number(param number)' + # }, + # 'Decides the output for a given number' + # ] + # } + + # def test_cross_package_hover(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" + # line, col = 5, 31 + # result = fizzbuzz_workspace.hover(uri, line, col) + # assert result == { + # 'contents': [ + # { + # 'language': 'python', + # 'value': 'def should_fizz(param number)' + # }, + # 'Whether or not "fizz" should be printed for this number' + # ] + # } + + # def test_std_lib_hover(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/__main__.py" + # line, col = 5, 10 + # result = fizzbuzz_workspace.hover(uri, line, col) + # assert result == { + # 'contents': [ + # { + # 'language': 'python', + # 'value': 'def print(param value, param ..., param sep, param ' + # 'end, param file, param flush)' + # }, + # "print(value, ..., sep=' ', end='\\n', file=sys.stdout, " + # 'flush=False)\n' + # '\n' + # 'Prints the values to a stream, or to sys.stdout by default.\n' + # 'Optional keyword arguments:\n' + # 'file: a file-like object (stream); defaults to the current ' + # 'sys.stdout.\n' + # 'sep: string inserted between values, default a space.\n' + # 'end: string appended after the last value, default a newline.\n' + # 'flush: whether to forcibly flush the stream.' + # ] + # } + + # def test_local_defintion(self, fizzbuzz_workspace): + # uri = "/fizzbuzz_service/string_deciders/number_decision.py" + # line, col = 21, 21 + # result = fizzbuzz_workspace.definition(uri, line, col) + # assert result == [ + # { + # 'symbol': { + # 'package': { + # 'name': 'fizzbuzz_service' + # }, + # 'name': 'OutputDecision', + # 'container': 'fizzbuzz_service.string_deciders.number_decision', + # 'kind': 'class', + # 'file': 'number_decision.py', + # 'position': { + # 'line': 5, + # 'character': 6 + # } + # }, + # 'location': { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 5, + # 'character': 6 + # }, + # 'end': { + # 'line': 5, + # 'character': 20 + # } + # } + # } + # } + # ] + + # def test_local_package_cross_module_definition(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + # line, col = 4, 25 + # result = fizzbuzz_workspace.definition(uri, line, col) + # definition = { + # 'symbol': { + # 'package': { + # 'name': 'fizzbuzz_service' + # }, + # 'name': 'decide_output_for_number', + # 'container': 'fizzbuzz_service.string_deciders.number_decision', + # 'kind': 'def', + # 'file': 'number_decision.py', + # 'position': { + # 'line': 12, + # 'character': 4 + # } + # }, + # 'location': { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 12, + # 'character': 4 + # }, + # 'end': { + # 'line': 12, + # 'character': 28 + # } + # } + # } + # } + + # # from the import statement at the top of the file + # assignment = { + # 'symbol': { + # 'package': { + # 'name': 'fizzbuzz_service' + # }, + # 'name': 'decide_output_for_number', + # 'container': 'fizzbuzz_service.string_deciders.number_decider', + # 'kind': 'def', + # 'file': 'number_decider.py', + # 'position': { + # 'line': 0, + # 'character': 45 + # } + # }, + # 'location': { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + # 'range': { + # 'start': { + # 'line': 0, + # 'character': 45 + # }, + # 'end': { + # 'line': 0, + # 'character': 69 + # } + # } + # } + # } + + # assert len(result) == 2 + + # assert definition in result + # assert assignment in result + + # def test_cross_package_definition(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" + # line, col = 5, 57 + # result = fizzbuzz_workspace.definition(uri, line, col) + + # definition = { + # 'symbol': { + # 'package': { + # 'name': 'fizzbuzz_service' + # }, + # 'name': 'should_buzz', + # 'container': 'buzz.buzz_checker', + # 'kind': 'def', + # 'file': 'buzz_checker.py', + # 'position': { + # 'line': 0, + # 'character': 4 + # } + # }, + # 'location': { + # 'uri': 'file:///fizzbuzz_service/checkers/buzz/buzz_checker.py', + # 'range': { + # 'start': { + # 'line': 0, + # 'character': 4 + # }, + # 'end': { + # 'line': 0, + # 'character': 15 + # } + # } + # } + # } + + # # from the import statement at the top of the file + # assignment = { + # 'symbol': { + # 'package': { + # 'name': 'fizzbuzz_service' + # }, + # 'name': 'should_buzz', + # 'container': 'fizzbuzz_service.checkers.fizzbuzz.fizzbuzz_checker', + # 'kind': 'def', + # 'file': 'fizzbuzz_checker.py', + # 'position': { + # 'line': 1, + # 'character': 32 + # }, + # }, + # 'location': { + # 'uri': 'file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py', + # 'range': { + # 'start': { + # 'line': 1, + # 'character': 32 + # }, + # 'end': { + # 'line': 1, + # 'character': 43 + # } + # } + # } + # } + + # assert len(result) == 2 + + # assert definition in result + # assert assignment in result + + # def test_local_package_cross_module_import_definition(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + # line, col = 0, 14 + # result = fizzbuzz_workspace.definition(uri, line, col) + + # assert len(result) == 1 + # definition = result[0] + + # assert "symbol" in definition + # assert definition["symbol"] == { + # 'package': { + # 'name': 'fizzbuzz_service'}, + # 'name': 'number_decision', + # 'container': 'fizzbuzz_service.string_deciders.number_decision', + # 'kind': 'module', + # 'file': 'number_decision.py', + # 'position': { + # 'line': 0, + # 'character': 0}, + # } + + # assert "location" in definition + # location = definition["location"] + # assert "uri" in location + # assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/number_decision.py' + + # # TODO: In the case of a module, does the range have any meaning? + + # def test_cross_package_import_definition(self, fizzbuzz_workspace): + # uri = "file:///fizzbuzz_service/loopers/number_looper.py" + # line, col = 0, 31 + # result = fizzbuzz_workspace.definition(uri, line, col) + + # assert len(result) == 1 + # definition = result[0] + + # assert "symbol" in definition + # assert definition["symbol"] == { + # 'package': { + # 'name': 'fizzbuzz_service' + # }, + # 'name': 'string_deciders', + # 'container': 'fizzbuzz_service.string_deciders', + # 'kind': 'module', + # 'file': '__init__.py', + # 'position': { + # 'line': 0, + # 'character': 0 + # }, + # } + + # assert "location" in definition + # location = definition["location"] + # assert "uri" in location + # assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/__init__.py' + + # # TODO: In the case of a module, does the range have any meaning? def test_std_lib_definition(self, fizzbuzz_workspace): uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' @@ -385,147 +385,149 @@ def test_std_lib_definition(self, fizzbuzz_workspace): } } + import pdb + pdb.set_trace() assert len(result) == 2 assert definition in result assert assignment in result - def test_local_references(self, fizzbuzz_workspace): - uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' - line, col = 5, 13 - result = fizzbuzz_workspace.references(uri, line, col) - - references = [ - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { - 'line': 6, - 'character': 19 - }, - 'end': { - 'line': 6, - 'character': 33 - } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { - 'line': 9, - 'character': 19 - }, - 'end': { - 'line': 9, - 'character': 33 - } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - 'range': { - 'start': { - 'line': 12, - 'character': 19 - }, - 'end': { - 'line': 12, - 'character': 33 - } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 15, - 'character': 15 - }, - 'end': { - 'line': 15, - 'character': 29 - } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 18, - 'character': 15 - }, - 'end': { - 'line': 18, - 'character': 29 - } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 21, - 'character': 15 - }, - 'end': - { - 'line': 21, - 'character': 29 - } - } - }, - { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 23, - 'character': 11 - }, - 'end': { - 'line': 23, - 'character': 25 - } - } - }, - ] - - assert len(references) == len(result) - for ref in references: - assert ref in result - - def test_x_references(self, fizzbuzz_workspace): - result = fizzbuzz_workspace.x_references("enum", "Enum") - - import_ref = { - 'reference': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 0, - 'character': 0}, - 'end': { - 'line': 0, - 'character': 4}}}, - 'symbol': { - 'container': 'enum', - 'name': 'Enum'}} - - base_class_ref = { - 'reference': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 5, - 'character': 21}, - 'end': { - 'line': 5, - 'character': 25}}}, - 'symbol': { - 'container': 'enum', - 'name': 'Enum'}} - - assert len(result) == 2 - - assert import_ref in result - assert base_class_ref in result + # def test_local_references(self, fizzbuzz_workspace): + # uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' + # line, col = 5, 13 + # result = fizzbuzz_workspace.references(uri, line, col) + + # references = [ + # { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + # 'range': { + # 'start': { + # 'line': 6, + # 'character': 19 + # }, + # 'end': { + # 'line': 6, + # 'character': 33 + # } + # } + # }, + # { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + # 'range': { + # 'start': { + # 'line': 9, + # 'character': 19 + # }, + # 'end': { + # 'line': 9, + # 'character': 33 + # } + # } + # }, + # { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + # 'range': { + # 'start': { + # 'line': 12, + # 'character': 19 + # }, + # 'end': { + # 'line': 12, + # 'character': 33 + # } + # } + # }, + # { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 15, + # 'character': 15 + # }, + # 'end': { + # 'line': 15, + # 'character': 29 + # } + # } + # }, + # { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 18, + # 'character': 15 + # }, + # 'end': { + # 'line': 18, + # 'character': 29 + # } + # } + # }, + # { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 21, + # 'character': 15 + # }, + # 'end': + # { + # 'line': 21, + # 'character': 29 + # } + # } + # }, + # { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 23, + # 'character': 11 + # }, + # 'end': { + # 'line': 23, + # 'character': 25 + # } + # } + # }, + # ] + + # assert len(references) == len(result) + # for ref in references: + # assert ref in result + + # def test_x_references(self, fizzbuzz_workspace): + # result = fizzbuzz_workspace.x_references("enum", "Enum") + + # import_ref = { + # 'reference': { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 0, + # 'character': 0}, + # 'end': { + # 'line': 0, + # 'character': 4}}}, + # 'symbol': { + # 'container': 'enum', + # 'name': 'Enum'}} + + # base_class_ref = { + # 'reference': { + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + # 'range': { + # 'start': { + # 'line': 5, + # 'character': 21}, + # 'end': { + # 'line': 5, + # 'character': 25}}}, + # 'symbol': { + # 'container': 'enum', + # 'name': 'Enum'}} + + # assert len(result) == 2 + + # assert import_ref in result + # assert base_class_ref in result From da0ddb6334ea95ba4510545f56a3f244e2ecac25 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Mon, 5 Mar 2018 11:15:04 -0800 Subject: [PATCH 06/45] wip --- Pipfile | 2 +- Pipfile.lock | 75 ++++++++++------------------------- langserver/clone_workspace.py | 40 ++++++++----------- langserver/config.py | 11 ++--- langserver/jedi.py | 9 ++++- langserver/langserver.py | 3 +- 6 files changed, 52 insertions(+), 88 deletions(-) diff --git a/Pipfile b/Pipfile index 48fbf2f..b1de99a 100644 --- a/Pipfile +++ b/Pipfile @@ -12,7 +12,7 @@ opentracing = "*" lightstep = "*" "delegator.py" = "*" pipenv = "*" -jedi = {git = "git://github.com/sourcegraph/jedi.git", editable = true, ref = "dee138a06ce5568bfda1f88d77184223c5cf5b8d"} +jedi = {git = "git://github.com/davidhalter/jedi.git", editable = true, ref = "3c9aa9ef254dbf1aa5f55ed3a4ed9ec4c6d0bb5a"} [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 7f1ef47..0046817 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,20 +1,7 @@ { "_meta": { "hash": { - "sha256": "a60044d027ec228213683ce4cc0cfc7246552d4a800ad9c284756dea51651027" - }, - "host-environment-markers": { - "implementation_name": "cpython", - "implementation_version": "3.6.4", - "os_name": "posix", - "platform_machine": "x86_64", - "platform_python_implementation": "CPython", - "platform_release": "17.4.0", - "platform_system": "Darwin", - "platform_version": "Darwin Kernel Version 17.4.0: Sun Dec 17 09:19:54 PST 2017; root:xnu-4570.41.2~1/RELEASE_X86_64", - "python_full_version": "3.6.4", - "python_version": "3.6", - "sys_platform": "darwin" + "sha256": "49c740bc7aadcb217862ec6482768eb43546b003d4ade69ce68cef8c8b248954" }, "pipfile-spec": 6, "requires": { @@ -35,20 +22,6 @@ ], "version": "==2.2.1" }, - "certifi": { - "hashes": [ - "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", - "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" - ], - "version": "==2018.1.18" - }, - "chardet": { - "hashes": [ - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" - ], - "version": "==3.0.4" - }, "delegator.py": { "hashes": [ "sha256:58f3ea6fe36680e1d828e2e66e52844b826f186409dfee4436e42351b0e699fe", @@ -56,17 +29,10 @@ ], "version": "==0.1.0" }, - "idna": { - "hashes": [ - "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", - "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" - ], - "version": "==2.6" - }, "jedi": { "editable": true, - "git": "git://github.com/sourcegraph/jedi.git", - "ref": "dee138a06ce5568bfda1f88d77184223c5cf5b8d" + "git": "git://github.com/davidhalter/jedi.git", + "ref": "3c9aa9ef254dbf1aa5f55ed3a4ed9ec4c6d0bb5a" }, "jsonpickle": { "hashes": [ @@ -87,6 +53,13 @@ ], "version": "==1.3.0" }, + "parso": { + "hashes": [ + "sha256:a7bb86fe0844304869d1c08e8bd0e52be931228483025c422917411ab82d628a", + "sha256:5815f3fe254e5665f3c5d6f54f086c2502035cb631a91341591b5a564203cffb" + ], + "version": "==0.1.1" + }, "pew": { "hashes": [ "sha256:a5256586e169199cb2213225261b6bdd7b657c309dcf4a02b61994308b313d4d", @@ -103,9 +76,9 @@ }, "pipenv": { "hashes": [ - "sha256:ce6dbb305fb1f262dba0dcb50c06591e4d146f7bfe079cc9f0ce3f89c7516ae9" + "sha256:0796cb5078b00a26b9332a87ed137efa0c21e585dbf945d5943022e9b35e1663" ], - "version": "==10.1.2" + "version": "==11.0.2" }, "protobuf": { "hashes": [ @@ -127,18 +100,16 @@ ], "version": "==0.5.2" }, - "requests": { - "hashes": [ - "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", - "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" - ], - "version": "==2.18.4" - }, "requirements": { "editable": true, "git": "git://github.com/sourcegraph/requirements-parser.git", "ref": "69f1a9cb916b2995843c3ea9b988da46c9dd65c7" }, + "requirements-parser": { + "editable": true, + "git": "git://github.com/sourcegraph/requirements-parser.git", + "ref": "69f1a9cb916b2995843c3ea9b988da46c9dd65c7" + }, "six": { "hashes": [ "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", @@ -152,13 +123,6 @@ ], "version": "==0.10.0" }, - "urllib3": { - "hashes": [ - "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", - "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" - ], - "version": "==1.22" - }, "virtualenv": { "hashes": [ "sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1fc9fbb7b481e5c85b0", @@ -168,9 +132,10 @@ }, "virtualenv-clone": { "hashes": [ - "sha256:6b3be5cab59e455f08c9eda573d23006b7d6fb41fae974ddaa2b275c93cc4405" + "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7", + "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8" ], - "version": "==0.2.6" + "version": "==0.3.0" } }, "develop": { diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 43c7c7b..75db1a6 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -8,7 +8,7 @@ import json from functools import lru_cache from enum import Enum -import pathlib +from pathlib import Path log = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def __init__(self, fs: FileSystem, project_root: str, original_root_path: str= ""): self.fs = fs - self.PROJECT_ROOT = project_root + self.PROJECT_ROOT = Path(project_root) self.repo = None self.hash = None @@ -39,30 +39,27 @@ def __init__(self, fs: FileSystem, project_root: str, if self.hash: self.key = ".".join((self.key, self.hash)) - self.PYTHON_PATH = os.path.abspath(GlobalConfig.PYTHON_PATH) - self.CLONED_PROJECT_PATH = os.path.abspath(os.path.join( - GlobalConfig.CLONED_PROJECT_PATH, self.key)) + self.CLONED_PROJECT_PATH = GlobalConfig.CLONED_PROJECT_PATH / self.key - log.debug("Setting Python path to %s", self.PYTHON_PATH) log.debug("Setting Cloned Project path to %s", self.CLONED_PROJECT_PATH) # Clone the project from the provided filesystem into the local # cache - all_files = self.fs.walk(self.PROJECT_ROOT) - for file_path in all_files: - cache_file_path = self.project_to_cache_path(file_path) + for file_path in self.fs.walk(str(self.PROJECT_ROOT)): - os.makedirs(os.path.dirname(cache_file_path), exist_ok=True) + cache_file_path = self.CLONED_PROJECT_PATH / file_path.lstrip("/") + + cache_file_path.parent.mkdir(parents=True, exist_ok=True) file_contents = self.fs.open(file_path) - with open(cache_file_path, "w") as f: - f.write(file_contents) + cache_file_path.write_text(file_contents) @property @lru_cache() - def VENV_LOCATION(self): + def VENV_PATH(self): self.ensure_venv_created() - return self.run_command("pipenv --venv").out.rstrip() + venv_path = self.run_command("pipenv --venv").out.rstrip() + return Path(venv_path) def cleanup(self): log.info("Removing project's virtual environment %s", self.VENV_LOCATION) @@ -106,16 +103,15 @@ def remove_venv(self): self.run_command("pipenv --rm", no_prefix=True) def get_module_info(self, raw_module_path): - module_path = pathlib.Path(raw_module_path) + module_path = Path(raw_module_path) - import pdb - pdb.set_trace() + if self.CLONED_PROJECT_PATH in module_path.parents: + return (ModuleKind.PROJECT, module_path.relative_to(self.CLONED_PROJECT_PATH)) - sys_std_lib_path = pathlib.Path(self.PYTHON_PATH) - if sys_std_lib_path in module_path.parents: + if GlobalConfig.PYTHON_PATH in module_path.parents: return (ModuleKind.STANDARD_LIBRARY, path.relative_to(sys_std_lib_path)) - venv_path = pathlib.Path(self.VENV_LOCATION) / "lib" + venv_path = self.VENV_LOCATION / "lib" if venv_path in module_path.parents: # The python libraries in a venv are stored under # VENV_LOCATION/lib/(some_python_version) @@ -129,10 +125,6 @@ def get_module_info(self, raw_module_path): return (ModuleKind.EXTERNAL_DEPENDENCY, module_path.relative_to(venv_ext_packages_path)) return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(venv_lib_path)) - project_path = pathlib.Path(self.CLONED_PROJECT_PATH) - if project_path in module_path.parents: - return (ModuleKind.PROJECT, module_path.relative_to(project_path)) - return (ModuleKind.UNKNOWN, module_path) def get_package_information(self): diff --git a/langserver/config.py b/langserver/config.py index 475cd6f..ccff3a7 100644 --- a/langserver/config.py +++ b/langserver/config.py @@ -1,12 +1,13 @@ -import distutils +from distutils.sysconfig import get_python_lib +from pathlib import Path class GlobalConfig: # TODO: allow different Python stdlib versions per workspace? - PYTHON_PATH = distutils.sysconfig.get_python_lib(standard_lib=True) - PACKAGES_PARENT = "python-langserver-cache" - CLONED_PROJECT_PATH = "python-cloned-projects-cache" + PYTHON_PATH = Path(get_python_lib(standard_lib=True)).absolute() + PACKAGES_PARENT = Path("python-langserver-cache").absolute() + CLONED_PROJECT_PATH = Path("python-cloned-projects-cache").absolute() STDLIB_REPO_URL = "git://github.com/python/cpython" - STDLIB_SRC_PATH = "Lib" + STDLIB_SRC_PATH = Path("Lib") diff --git a/langserver/jedi.py b/langserver/jedi.py index c572588..df58c8a 100644 --- a/langserver/jedi.py +++ b/langserver/jedi.py @@ -33,10 +33,15 @@ def _new_script_impl(self, parent_span, *args, **kwargs): if self.workspace is not None: path = self.workspace.project_to_cache_path(path) - venv_path = self.workspace.VENV_LOCATION + + environment = None + for env in jedi.find_virtualenvs([self.workspace.VENV_PATH], safe=False): + if env._base_path == self.workspace.VENV_PATH: + environment = env + break kwargs.update( - venv_path=venv_path, + environment=environment, path=path ) diff --git a/langserver/langserver.py b/langserver/langserver.py index 0007ff5..f960cce 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -368,6 +368,7 @@ def serve_x_definition(self, request): if kind is not ModuleKind.UNKNOWN: if kind == ModuleKind.STANDARD_LIBRARY: filename = module_path.name + module_path = GlobalConfig.STDLIB_SRC_PATH / module_path symbol_name = "" symbol_kind = "" if d.description: @@ -380,7 +381,7 @@ def serve_x_definition(self, request): "name": symbol_name, "container": d.full_name, "kind": symbol_kind, - "path": str(GlobalConfig.STDLIB_SRC_PATH / module_path), + "path": str(module_path), "file": filename } else: From 498caf28e492d35a2aea5bc8ee1539ccaa9a6853 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 14:02:20 -0800 Subject: [PATCH 07/45] add doc to get_module_info --- langserver/clone_workspace.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 75db1a6..6e64597 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -102,14 +102,26 @@ def ensure_venv_created(self): def remove_venv(self): self.run_command("pipenv --rm", no_prefix=True) - def get_module_info(self, raw_module_path): - module_path = Path(raw_module_path) + def get_module_info(self, raw_jedi_module_path): + """ + Given an absolute module path provided from jedi, + returns a tuple of (module_kind, rel_path) where: + + module_kind: The category that the module belongs to + (module is declared inside the project, module is a std_lib module, etc.) + + rel_path: The path of the module relative to the context + which it's defined in. e.x: if module_kind == 'PROJECT', + rel_path is the path of the module relative to the project's root. + """ + + module_path = Path(raw_jedi_module_path) if self.CLONED_PROJECT_PATH in module_path.parents: return (ModuleKind.PROJECT, module_path.relative_to(self.CLONED_PROJECT_PATH)) if GlobalConfig.PYTHON_PATH in module_path.parents: - return (ModuleKind.STANDARD_LIBRARY, path.relative_to(sys_std_lib_path)) + return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(GlobalConfig.PYTHON_PATH)) venv_path = self.VENV_LOCATION / "lib" if venv_path in module_path.parents: @@ -119,10 +131,11 @@ def get_module_info(self, raw_module_path): python_version = module_path.relative_to(venv_path).parts[0] venv_lib_path = venv_path / python_version - venv_ext_packages_path = venv_lib_path / "site-packages" + ext_dep_path = venv_lib_path / "site-packages" + + if ext_dep_path in module_path.parents: + return (ModuleKind.EXTERNAL_DEPENDENCY, module_path.relative_to(ext_dep_path)) - if venv_ext_packages_path in module_path.parents: - return (ModuleKind.EXTERNAL_DEPENDENCY, module_path.relative_to(venv_ext_packages_path)) return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(venv_lib_path)) return (ModuleKind.UNKNOWN, module_path) From bbe5a09b3e5ff1c1eec0347b14bf83ae843948ce Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 14:43:29 -0800 Subject: [PATCH 08/45] fix venv_location reference --- langserver/clone_workspace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 6e64597..c7351d4 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -62,7 +62,7 @@ def VENV_PATH(self): return Path(venv_path) def cleanup(self): - log.info("Removing project's virtual environment %s", self.VENV_LOCATION) + log.info("Removing project's virtual environment %s", self.VENV_PATH) self.remove_venv() log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) @@ -123,7 +123,7 @@ def get_module_info(self, raw_jedi_module_path): if GlobalConfig.PYTHON_PATH in module_path.parents: return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(GlobalConfig.PYTHON_PATH)) - venv_path = self.VENV_LOCATION / "lib" + venv_path = self.VENV_PATH / "lib" if venv_path in module_path.parents: # The python libraries in a venv are stored under # VENV_LOCATION/lib/(some_python_version) From 2826abb0b2ffb64702d79a5d561f9db2956e6e8d Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 14:44:13 -0800 Subject: [PATCH 09/45] delete a lot of the xdef logic for now, use cloneworkspace outside test --- langserver/langserver.py | 105 ++++++++++++--------------------------- 1 file changed, 33 insertions(+), 72 deletions(-) diff --git a/langserver/langserver.py b/langserver/langserver.py index f960cce..19dd363 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -172,12 +172,16 @@ def serve_initialize(self, request): if isinstance(p, list): pip_args = p else: - log.error("pipArgs (%s) found, but was not a list, so ignoring", str(p)) + log.error( + "pipArgs (%s) found, but was not a list, so ignoring", str(p)) # Sourcegraph also passes in a rootUri which has commit information originalRootUri = params.get("originalRootUri") or params.get( "originalRootPath") or "" - self.workspace = Workspace(self.fs, self.root_path, originalRootUri, pip_args) + # self.workspace = Workspace( + # self.fs, self.root_path, originalRootUri, pip_args) + self.workspace = CloneWorkspace( + self.fs, self.root_path, originalRootUri) return { "capabilities": { @@ -354,83 +358,39 @@ def serve_x_definition(self, request): return results for d in defs: - kind, module_path = ModuleKind.UNKNOWN, "" - if d.module_path: - kind, module_path = self.workspace.get_module_info( - d.module_path) + # TODO: handle case where a def doesn't have a module_path + if not d.module_path: + continue + + module_kind, rel_module_path = self.workspace.get_module_info( + d.module_path) + + if module_kind != ModuleKind.PROJECT: + # only handle internal defs for now + continue if (not d.is_definition() or d.line is None or d.column is None): continue - symbol_locator = {"symbol": None, "location": None} - - if kind is not ModuleKind.UNKNOWN: - if kind == ModuleKind.STANDARD_LIBRARY: - filename = module_path.name - module_path = GlobalConfig.STDLIB_SRC_PATH / module_path - symbol_name = "" - symbol_kind = "" - if d.description: - symbol_name, symbol_kind = LangServer.name_and_kind( - d.description) - symbol_locator["symbol"] = { - "package": { - "name": "cpython", - }, - "name": symbol_name, - "container": d.full_name, - "kind": symbol_kind, - "path": str(module_path), - "file": filename - } - else: - # the module path doesn't map onto the repository structure - # because we're not fully installing - # dependency packages, so don't include it in the symbol - # descriptor - filename = module_path.name - symbol_name = "" - symbol_kind = "" - if d.description: - symbol_name, symbol_kind = LangServer.name_and_kind( - d.description) - symbol_locator["symbol"] = { - "package": { - "name": d.full_name.split(".")[0], - }, - "name": symbol_name, - "container": d.full_name, - "kind": symbol_kind, - "file": filename - } + location = { + # put a "/" shim at the front to make the path + # seem like an absolute one inside the project + "uri": "file://" + str("/" / rel_module_path), - if (d.is_definition() and - d.line is not None and d.column is not None): - location = { - # TODO(renfred) determine why d.module_path is empty. - "uri": "file://" + (str(module_path) or path), - "range": { - "start": { - "line": d.line - 1, - "character": d.column, - }, - "end": { - "line": d.line - 1, - "character": d.column + len(d.name), - }, + "range": { + "start": { + "line": d.line - 1, + "character": d.column, }, - } - # add a position hint in case this eventually gets passed to an - # operation that could use it - if symbol_locator["symbol"]: - symbol_locator["symbol"]["position"] = location["range"][ - "start"] - - # set the full location if the definition is in this workspace - if not kind in [ModuleKind.UNKNOWN, ModuleKind.EXTERNAL_DEPENDENCY]: - symbol_locator["location"] = location + "end": { + "line": d.line - 1, + "character": d.column + len(d.name), + }, + }, + } + symbol_locator = {"symbol": None, "location": location} results.append(symbol_locator) unique_results = [] @@ -701,7 +661,8 @@ def main(): parser.add_argument( "--addr", default=4389, help="server listen (tcp)", type=int) parser.add_argument("--debug", action="store_true") - parser.add_argument("--lightstep_token", default=os.environ.get("LIGHTSTEP_ACCESS_TOKEN")) + parser.add_argument("--lightstep_token", + default=os.environ.get("LIGHTSTEP_ACCESS_TOKEN")) parser.add_argument("--python_path") args = parser.parse_args() From b52e0c804f10508b31d8cbfa716b428dae44f461 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 14:45:04 -0800 Subject: [PATCH 10/45] comment out pip args logic for now --- langserver/langserver.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/langserver/langserver.py b/langserver/langserver.py index 19dd363..a378cfe 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -164,16 +164,16 @@ def serve_initialize(self, request): else: self.fs = LocalFileSystem() - pip_args = [] - if "initializationOptions" in params: - initOps = params["initializationOptions"] - if isinstance(initOps, dict) and "pipArgs" in initOps: - p = initOps["pipArgs"] - if isinstance(p, list): - pip_args = p - else: - log.error( - "pipArgs (%s) found, but was not a list, so ignoring", str(p)) + # pip_args = [] + # if "initializationOptions" in params: + # initOps = params["initializationOptions"] + # if isinstance(initOps, dict) and "pipArgs" in initOps: + # p = initOps["pipArgs"] + # if isinstance(p, list): + # pip_args = p + # else: + # log.error( + # "pipArgs (%s) found, but was not a list, so ignoring", str(p)) # Sourcegraph also passes in a rootUri which has commit information originalRootUri = params.get("originalRootUri") or params.get( From 25304ad1b69f9350d5840b8be07dda973b7783fd Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 14:47:41 -0800 Subject: [PATCH 11/45] uncomment hover tests --- test/test_fizzbuzz.py | 128 +++++++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/test/test_fizzbuzz.py b/test/test_fizzbuzz.py index eea66c1..e3facec 100644 --- a/test/test_fizzbuzz.py +++ b/test/test_fizzbuzz.py @@ -34,73 +34,73 @@ class TestFizzBuzzWorkspace: # "cpython" # } - # def test_local_hover(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/loopers/number_looper.py" - # line, col = 2, 7 - # result = fizzbuzz_workspace.hover(uri, line, col) - # assert result == { - # 'contents': [ - # { - # 'language': 'python', - # 'value': 'class NumberLooper(param start, param end)' - # }, - # 'Very important class that is capable of gathering all the number strings ' - # 'in [start, end)' - # ] - # } + def test_local_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/loopers/number_looper.py" + line, col = 2, 7 + result = fizzbuzz_workspace.hover(uri, line, col) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'class NumberLooper(param start, param end)' + }, + 'Very important class that is capable of gathering all the number strings ' + 'in [start, end)' + ] + } - # def test_local_package_cross_module_hover(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - # line, col = 4, 22 - # result = fizzbuzz_workspace.hover(uri, line, col) - - # assert result == { - # 'contents': [ - # { - # 'language': 'python', - # 'value': 'def decide_output_for_number(param number)' - # }, - # 'Decides the output for a given number' - # ] - # } + def test_local_package_cross_module_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + line, col = 4, 22 + result = fizzbuzz_workspace.hover(uri, line, col) - # def test_cross_package_hover(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" - # line, col = 5, 31 - # result = fizzbuzz_workspace.hover(uri, line, col) - # assert result == { - # 'contents': [ - # { - # 'language': 'python', - # 'value': 'def should_fizz(param number)' - # }, - # 'Whether or not "fizz" should be printed for this number' - # ] - # } + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'def decide_output_for_number(param number)' + }, + 'Decides the output for a given number' + ] + } - # def test_std_lib_hover(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/__main__.py" - # line, col = 5, 10 - # result = fizzbuzz_workspace.hover(uri, line, col) - # assert result == { - # 'contents': [ - # { - # 'language': 'python', - # 'value': 'def print(param value, param ..., param sep, param ' - # 'end, param file, param flush)' - # }, - # "print(value, ..., sep=' ', end='\\n', file=sys.stdout, " - # 'flush=False)\n' - # '\n' - # 'Prints the values to a stream, or to sys.stdout by default.\n' - # 'Optional keyword arguments:\n' - # 'file: a file-like object (stream); defaults to the current ' - # 'sys.stdout.\n' - # 'sep: string inserted between values, default a space.\n' - # 'end: string appended after the last value, default a newline.\n' - # 'flush: whether to forcibly flush the stream.' - # ] - # } + def test_cross_package_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" + line, col = 5, 31 + result = fizzbuzz_workspace.hover(uri, line, col) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'def should_fizz(param number)' + }, + 'Whether or not "fizz" should be printed for this number' + ] + } + + def test_std_lib_hover(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/__main__.py" + line, col = 5, 10 + result = fizzbuzz_workspace.hover(uri, line, col) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'def print(param value, param ..., param sep, param ' + 'end, param file, param flush)' + }, + "print(value, ..., sep=' ', end='\\n', file=sys.stdout, " + 'flush=False)\n' + '\n' + 'Prints the values to a stream, or to sys.stdout by default.\n' + 'Optional keyword arguments:\n' + 'file: a file-like object (stream); defaults to the current ' + 'sys.stdout.\n' + 'sep: string inserted between values, default a space.\n' + 'end: string appended after the last value, default a newline.\n' + 'flush: whether to forcibly flush the stream.' + ] + } # def test_local_defintion(self, fizzbuzz_workspace): # uri = "/fizzbuzz_service/string_deciders/number_decision.py" From f82f8a0710e92d0dd396bde8220f2c7b542ff384 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:10:02 -0800 Subject: [PATCH 12/45] add local definition tests back for fizzbuzz --- test/test_fizzbuzz.py | 407 ++++++++++++++++-------------------------- 1 file changed, 156 insertions(+), 251 deletions(-) diff --git a/test/test_fizzbuzz.py b/test/test_fizzbuzz.py index e3facec..6b0efd2 100644 --- a/test/test_fizzbuzz.py +++ b/test/test_fizzbuzz.py @@ -102,141 +102,164 @@ def test_std_lib_hover(self, fizzbuzz_workspace): ] } - # def test_local_defintion(self, fizzbuzz_workspace): - # uri = "/fizzbuzz_service/string_deciders/number_decision.py" - # line, col = 21, 21 - # result = fizzbuzz_workspace.definition(uri, line, col) - # assert result == [ - # { - # 'symbol': { - # 'package': { - # 'name': 'fizzbuzz_service' - # }, - # 'name': 'OutputDecision', - # 'container': 'fizzbuzz_service.string_deciders.number_decision', - # 'kind': 'class', - # 'file': 'number_decision.py', - # 'position': { - # 'line': 5, - # 'character': 6 - # } - # }, - # 'location': { - # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - # 'range': { - # 'start': { - # 'line': 5, - # 'character': 6 - # }, - # 'end': { - # 'line': 5, - # 'character': 20 - # } - # } - # } - # } - # ] + def test_local_defintion(self, fizzbuzz_workspace): + uri = "/fizzbuzz_service/string_deciders/number_decision.py" + line, col = 21, 21 + result = fizzbuzz_workspace.definition(uri, line, col) - # def test_local_package_cross_module_definition(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - # line, col = 4, 25 - # result = fizzbuzz_workspace.definition(uri, line, col) - # definition = { - # 'symbol': { - # 'package': { - # 'name': 'fizzbuzz_service' - # }, - # 'name': 'decide_output_for_number', - # 'container': 'fizzbuzz_service.string_deciders.number_decision', - # 'kind': 'def', - # 'file': 'number_decision.py', - # 'position': { - # 'line': 12, - # 'character': 4 - # } - # }, - # 'location': { - # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - # 'range': { - # 'start': { - # 'line': 12, - # 'character': 4 - # }, - # 'end': { - # 'line': 12, - # 'character': 28 - # } - # } - # } - # } + assert len(result) == 1 + definition = result[0] - # # from the import statement at the top of the file - # assignment = { - # 'symbol': { - # 'package': { - # 'name': 'fizzbuzz_service' - # }, - # 'name': 'decide_output_for_number', - # 'container': 'fizzbuzz_service.string_deciders.number_decider', - # 'kind': 'def', - # 'file': 'number_decider.py', - # 'position': { - # 'line': 0, - # 'character': 45 - # } - # }, - # 'location': { - # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', - # 'range': { - # 'start': { - # 'line': 0, - # 'character': 45 - # }, - # 'end': { - # 'line': 0, - # 'character': 69 - # } - # } - # } - # } + assert "location" in definition + assert definition["location"] == { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 5, + 'character': 6 + }, + 'end': { + 'line': 5, + 'character': 20 + } + } + } - # assert len(result) == 2 + def test_local_package_cross_module_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + line, col = 4, 25 + result = fizzbuzz_workspace.definition(uri, line, col) - # assert definition in result - # assert assignment in result + assert len(result) == 2 + assert all(["location" in d for d in result]) + + result_locations = [d["location"] for d in result] + + definition_location = { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', + 'range': { + 'start': { + 'line': 12, + 'character': 4 + }, + 'end': { + 'line': 12, + 'character': 28 + } + } + } + + # from the import statement at the top of the file + assignment_location = { + 'uri': 'file:///fizzbuzz_service/string_deciders/number_decider.py', + 'range': { + 'start': { + 'line': 0, + 'character': 45 + }, + 'end': { + 'line': 0, + 'character': 69 + } + } + } + + assert definition_location in result_locations + assert assignment_location in result_locations + + def test_cross_package_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" + line, col = 5, 57 + result = fizzbuzz_workspace.definition(uri, line, col) - # def test_cross_package_definition(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py" - # line, col = 5, 57 + assert len(result) == 2 + assert all(["location" in d for d in result]) + + result_locations = [d["location"] for d in result] + + definition_location = { + 'uri': 'file:///fizzbuzz_service/checkers/buzz/buzz_checker.py', + 'range': { + 'start': { + 'line': 0, + 'character': 4 + }, + 'end': { + 'line': 0, + 'character': 15 + } + } + } + + # from the import statement at the top of the file + assignment_location = { + 'uri': 'file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py', + 'range': { + 'start': { + 'line': 1, + 'character': 32 + }, + 'end': { + 'line': 1, + 'character': 43 + } + } + } + + assert definition_location in result_locations + assert assignment_location in result_locations + + def test_local_package_cross_module_import_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" + line, col = 0, 14 + result = fizzbuzz_workspace.definition(uri, line, col) + + assert len(result) == 1 + definition = result[0] + + assert "location" in definition + location = definition["location"] + assert "uri" in location + assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/number_decision.py' + + # TODO: In the case of a module, does the range have any meaning? + + def test_cross_package_import_definition(self, fizzbuzz_workspace): + uri = "file:///fizzbuzz_service/loopers/number_looper.py" + line, col = 0, 31 + result = fizzbuzz_workspace.definition(uri, line, col) + + assert len(result) == 1 + definition = result[0] + + assert "location" in definition + location = definition["location"] + assert "uri" in location + assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/__init__.py' + + # TODO: In the case of a module, does the range have any meaning? + + # def test_std_lib_definition(self, fizzbuzz_workspace): + # uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' + # line, col = 5, 23 # result = fizzbuzz_workspace.definition(uri, line, col) # definition = { # 'symbol': { # 'package': { - # 'name': 'fizzbuzz_service' + # 'name': 'cpython' # }, - # 'name': 'should_buzz', - # 'container': 'buzz.buzz_checker', - # 'kind': 'def', - # 'file': 'buzz_checker.py', + # 'name': 'Enum', + # 'container': 'enum', + # 'kind': 'class', + # 'path': 'Lib/enum.py', + # 'file': 'enum.py', # 'position': { - # 'line': 0, - # 'character': 4 - # } + # 'line': 508, + # 'character': 6 + # }, # }, - # 'location': { - # 'uri': 'file:///fizzbuzz_service/checkers/buzz/buzz_checker.py', - # 'range': { - # 'start': { - # 'line': 0, - # 'character': 4 - # }, - # 'end': { - # 'line': 0, - # 'character': 15 - # } - # } - # } + # 'location': None # } # # from the import statement at the top of the file @@ -245,25 +268,25 @@ def test_std_lib_hover(self, fizzbuzz_workspace): # 'package': { # 'name': 'fizzbuzz_service' # }, - # 'name': 'should_buzz', - # 'container': 'fizzbuzz_service.checkers.fizzbuzz.fizzbuzz_checker', - # 'kind': 'def', - # 'file': 'fizzbuzz_checker.py', + # 'name': 'Enum', + # 'container': 'fizzbuzz_service.string_deciders.number_decision', + # 'kind': 'class', + # 'file': 'number_decision.py', # 'position': { - # 'line': 1, - # 'character': 32 + # 'line': 0, + # 'character': 17 # }, # }, # 'location': { - # 'uri': 'file:///fizzbuzz_service/checkers/fizzbuzz/fizzbuzz_checker.py', + # 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', # 'range': { # 'start': { - # 'line': 1, - # 'character': 32 + # 'line': 0, + # 'character': 17 # }, # 'end': { - # 'line': 1, - # 'character': 43 + # 'line': 0, + # 'character': 21 # } # } # } @@ -274,124 +297,6 @@ def test_std_lib_hover(self, fizzbuzz_workspace): # assert definition in result # assert assignment in result - # def test_local_package_cross_module_import_definition(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/string_deciders/number_decider.py" - # line, col = 0, 14 - # result = fizzbuzz_workspace.definition(uri, line, col) - - # assert len(result) == 1 - # definition = result[0] - - # assert "symbol" in definition - # assert definition["symbol"] == { - # 'package': { - # 'name': 'fizzbuzz_service'}, - # 'name': 'number_decision', - # 'container': 'fizzbuzz_service.string_deciders.number_decision', - # 'kind': 'module', - # 'file': 'number_decision.py', - # 'position': { - # 'line': 0, - # 'character': 0}, - # } - - # assert "location" in definition - # location = definition["location"] - # assert "uri" in location - # assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/number_decision.py' - - # # TODO: In the case of a module, does the range have any meaning? - - # def test_cross_package_import_definition(self, fizzbuzz_workspace): - # uri = "file:///fizzbuzz_service/loopers/number_looper.py" - # line, col = 0, 31 - # result = fizzbuzz_workspace.definition(uri, line, col) - - # assert len(result) == 1 - # definition = result[0] - - # assert "symbol" in definition - # assert definition["symbol"] == { - # 'package': { - # 'name': 'fizzbuzz_service' - # }, - # 'name': 'string_deciders', - # 'container': 'fizzbuzz_service.string_deciders', - # 'kind': 'module', - # 'file': '__init__.py', - # 'position': { - # 'line': 0, - # 'character': 0 - # }, - # } - - # assert "location" in definition - # location = definition["location"] - # assert "uri" in location - # assert location["uri"] == 'file:///fizzbuzz_service/string_deciders/__init__.py' - - # # TODO: In the case of a module, does the range have any meaning? - - def test_std_lib_definition(self, fizzbuzz_workspace): - uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' - line, col = 5, 23 - result = fizzbuzz_workspace.definition(uri, line, col) - - definition = { - 'symbol': { - 'package': { - 'name': 'cpython' - }, - 'name': 'Enum', - 'container': 'enum', - 'kind': 'class', - 'path': 'Lib/enum.py', - 'file': 'enum.py', - 'position': { - 'line': 508, - 'character': 6 - }, - }, - 'location': None - } - - # from the import statement at the top of the file - assignment = { - 'symbol': { - 'package': { - 'name': 'fizzbuzz_service' - }, - 'name': 'Enum', - 'container': 'fizzbuzz_service.string_deciders.number_decision', - 'kind': 'class', - 'file': 'number_decision.py', - 'position': { - 'line': 0, - 'character': 17 - }, - }, - 'location': { - 'uri': 'file:///fizzbuzz_service/string_deciders/number_decision.py', - 'range': { - 'start': { - 'line': 0, - 'character': 17 - }, - 'end': { - 'line': 0, - 'character': 21 - } - } - } - } - - import pdb - pdb.set_trace() - assert len(result) == 2 - - assert definition in result - assert assignment in result - # def test_local_references(self, fizzbuzz_workspace): # uri = 'file:///fizzbuzz_service/string_deciders/number_decision.py' # line, col = 5, 13 From 363db5724c70560a88c87f6461294f143cb0790d Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:11:17 -0800 Subject: [PATCH 13/45] fix lint --- langserver/langserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langserver/langserver.py b/langserver/langserver.py index a378cfe..9120ba0 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -11,7 +11,7 @@ from .fs import LocalFileSystem, RemoteFileSystem from .jedi import RemoteJedi from .jsonrpc import JSONRPC2Connection, ReadWriter, TCPReadWriter -from .workspace import Workspace +# from .workspace import Workspace from .symbols import extract_symbols, workspace_symbols from .definitions import targeted_symbol from .references import get_references From 7cfb6bfa5be035cc4ef3f117ae93290527227a93 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:15:50 -0800 Subject: [PATCH 14/45] rm unused cache to project path, clean logic in project to cache --- langserver/clone_workspace.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index c7351d4..166b8af 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -76,22 +76,12 @@ def project_to_cache_path(self, project_path): e.x.: '/a/b.py' -> 'python-cloned-projects-cache/project_name/a/b.py' """ # strip the leading '/' so that we can join it properly - file_path = os.path.relpath(project_path, "/") + file_path = project_path + if file_path.startswith("/"): + file_path = file_path[1:] return os.path.join(self.CLONED_PROJECT_PATH, file_path) - def cache_path_to_project_path(self, cache_path): - """ - Translates a path in the cache to the equivalent path in - the project. - - e.x.: 'python-cloned-projects-cache/project_name/a/b.py' -> '/a/b.py' - """ - file_path = os.path.relpath(cache_path, self.CLONED_PROJECT_PATH) - if not file_path.startswith("/"): - file_path = "/" + file_path - return file_path - def ensure_venv_created(self): ''' This runs a noop pipenv command, which will From e1ddfb38cde1c19db632e1b55a37288d61442691 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:20:09 -0800 Subject: [PATCH 15/45] re-order clone_workspace methods --- langserver/clone_workspace.py | 76 +++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 166b8af..3c7e925 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -54,44 +54,6 @@ def __init__(self, fs: FileSystem, project_root: str, file_contents = self.fs.open(file_path) cache_file_path.write_text(file_contents) - @property - @lru_cache() - def VENV_PATH(self): - self.ensure_venv_created() - venv_path = self.run_command("pipenv --venv").out.rstrip() - return Path(venv_path) - - def cleanup(self): - log.info("Removing project's virtual environment %s", self.VENV_PATH) - self.remove_venv() - - log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) - rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) - - def project_to_cache_path(self, project_path): - """ - Translates a path from the root of the project to the equivalent path in - the local cache. - - e.x.: '/a/b.py' -> 'python-cloned-projects-cache/project_name/a/b.py' - """ - # strip the leading '/' so that we can join it properly - file_path = project_path - if file_path.startswith("/"): - file_path = file_path[1:] - - return os.path.join(self.CLONED_PROJECT_PATH, file_path) - - def ensure_venv_created(self): - ''' - This runs a noop pipenv command, which will - create the venv if it doesn't exist as a side effect. - ''' - self.run_command("true") - - def remove_venv(self): - self.run_command("pipenv --rm", no_prefix=True) - def get_module_info(self, raw_jedi_module_path): """ Given an absolute module path provided from jedi, @@ -183,6 +145,44 @@ def external_dependencies(self): return out + def project_to_cache_path(self, project_path): + """ + Translates a path from the root of the project to the equivalent path in + the local cache. + + e.x.: '/a/b.py' -> 'python-cloned-projects-cache/project_name/a/b.py' + """ + # strip the leading '/' so that we can join it properly + file_path = project_path + if file_path.startswith("/"): + file_path = file_path[1:] + + return os.path.join(self.CLONED_PROJECT_PATH, file_path) + + @property + @lru_cache() + def VENV_PATH(self): + self.ensure_venv_created() + venv_path = self.run_command("pipenv --venv").out.rstrip() + return Path(venv_path) + + def cleanup(self): + log.info("Removing project's virtual environment %s", self.VENV_PATH) + self.remove_venv() + + log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) + rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) + + def ensure_venv_created(self): + ''' + This runs a noop pipenv command, which will + create the venv if it doesn't exist as a side effect. + ''' + self.run_command("true") + + def remove_venv(self): + self.run_command("pipenv --rm", no_prefix=True) + def run_command(self, command, no_prefix=False, **kwargs): ''' Runs the given command inside the context From 54f80ddc71ff454191e7432a7d05b28b4b0a1bcd Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:20:46 -0800 Subject: [PATCH 16/45] remove unused x-def related methods, distracting for now --- langserver/clone_workspace.py | 53 ---------------------------------- test/.cache/v/cache/lastfailed | 5 ---- 2 files changed, 58 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 3c7e925..e60c712 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -92,59 +92,6 @@ def get_module_info(self, raw_jedi_module_path): return (ModuleKind.UNKNOWN, module_path) - def get_package_information(self): - project_packages = self.project_packages() - dependencies = self.external_dependencies() - - out = [] - for package in project_packages: - out.append({ - "package": { - "name": package - }, - "dependencies": dependencies - }) - return out - - @lru_cache() - def project_packages(self): - ''' - Provides a list of all packages declared in the project - ''' - script = [ - "import json", - "import setuptools", - "pkgs = setuptools.find_packages()", - "print(json.dumps(pkgs))" - ] - - c = self.run_command("python -c '{}'".format(";".join(script))) - return json.loads(c.out) - - def external_dependencies(self): - ''' - Provides a list of third party packages that the - project depends on. - ''' - deps = json.loads(self.run_command( - "pip list --local --format json").out) - out = [ - { - # TODO - is this ever not a dependency? - "attributes": { - "name": "cpython", - "repoURL": "git://github.com/python/cpython" - } - } - ] - - for dep in deps: - dep_name = dep["name"] - if dep_name not in set(["pip", "wheel"]): - out.append({"attributes": {"name": dep_name}}) - - return out - def project_to_cache_path(self, project_path): """ Translates a path from the root of the project to the equivalent path in diff --git a/test/.cache/v/cache/lastfailed b/test/.cache/v/cache/lastfailed index 2f3efc7..b35d3ee 100644 --- a/test/.cache/v/cache/lastfailed +++ b/test/.cache/v/cache/lastfailed @@ -5,11 +5,6 @@ "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data1]": true, "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data2]": true, "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data3]": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_cross_package_definition": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_cross_package_import_definition": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_defintion": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_package_cross_module_definition": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_package_cross_module_import_definition": true, "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_references": true, "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_std_lib_definition": true, "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_x_references": true, From 10245a7a1a54241d55f9d06350d684e2d918ef27 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:22:24 -0800 Subject: [PATCH 17/45] project_to_cache: change doc string to make both abolute --- langserver/clone_workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index e60c712..5da9497 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -97,7 +97,7 @@ def project_to_cache_path(self, project_path): Translates a path from the root of the project to the equivalent path in the local cache. - e.x.: '/a/b.py' -> 'python-cloned-projects-cache/project_name/a/b.py' + e.x.: '/a/b.py' -> '/python-cloned-projects-cache/project_name/a/b.py' """ # strip the leading '/' so that we can join it properly file_path = project_path From 5be8955b1e6f406acf90478fd8500f6e70873edf Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:26:33 -0800 Subject: [PATCH 18/45] add docstring for venv_path --- langserver/clone_workspace.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 5da9497..01f2a38 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -109,6 +109,9 @@ def project_to_cache_path(self, project_path): @property @lru_cache() def VENV_PATH(self): + """ + The absolute path of the virtual environment created for this workspace. + """ self.ensure_venv_created() venv_path = self.run_command("pipenv --venv").out.rstrip() return Path(venv_path) From 39bcd1412f0b6ad8328850ed6d057c966c4a018b Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:30:46 -0800 Subject: [PATCH 19/45] fix lint --- langserver/clone_workspace.py | 1 - 1 file changed, 1 deletion(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 01f2a38..90f48df 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -5,7 +5,6 @@ from .fs import FileSystem from shutil import rmtree import delegator -import json from functools import lru_cache from enum import Enum from pathlib import Path From cc78c3382d1641cdeb57941220db8440b3b4a646 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 15:34:00 -0800 Subject: [PATCH 20/45] simplify project_to_cache, use in constructor --- langserver/clone_workspace.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 90f48df..252b8a3 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -1,6 +1,4 @@ import logging -import os -import os.path from .config import GlobalConfig from .fs import FileSystem from shutil import rmtree @@ -47,7 +45,7 @@ def __init__(self, fs: FileSystem, project_root: str, # cache for file_path in self.fs.walk(str(self.PROJECT_ROOT)): - cache_file_path = self.CLONED_PROJECT_PATH / file_path.lstrip("/") + cache_file_path = self.project_to_cache_path(file_path) cache_file_path.parent.mkdir(parents=True, exist_ok=True) file_contents = self.fs.open(file_path) @@ -98,12 +96,9 @@ def project_to_cache_path(self, project_path): e.x.: '/a/b.py' -> '/python-cloned-projects-cache/project_name/a/b.py' """ - # strip the leading '/' so that we can join it properly - file_path = project_path - if file_path.startswith("/"): - file_path = file_path[1:] - return os.path.join(self.CLONED_PROJECT_PATH, file_path) + # strip the leading '/' so that we can join it properly + return self.CLONED_PROJECT_PATH / project_path.lstrip("/") @property @lru_cache() From 7bf3c6ab6bbf2383ea3d1869d825aaa2bb9d0338 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 16:23:51 -0800 Subject: [PATCH 21/45] wip - install third party deps --- langserver/clone_workspace.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 252b8a3..fa1825a 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -43,7 +43,10 @@ def __init__(self, fs: FileSystem, project_root: str, # Clone the project from the provided filesystem into the local # cache + existing_pipfile = False for file_path in self.fs.walk(str(self.PROJECT_ROOT)): + if file_path.endswith("Pipfile"): + existing_pipfile = True cache_file_path = self.project_to_cache_path(file_path) @@ -51,6 +54,16 @@ def __init__(self, fs: FileSystem, project_root: str, file_contents = self.fs.open(file_path) cache_file_path.write_text(file_contents) + # Install 3rd party deps + + # pipenv creates the Pipfile automatically whenever it does anything - + # only install if the project had one to begin with + if existing_pipfile: + self._install_pipenv() + + self._install_pip() + self._install_setup_py() + def get_module_info(self, raw_jedi_module_path): """ Given an absolute module path provided from jedi, @@ -100,6 +113,19 @@ def project_to_cache_path(self, project_path): # strip the leading '/' so that we can join it properly return self.CLONED_PROJECT_PATH / project_path.lstrip("/") + def _install_pipenv(self): + if (self.CLONED_PROJECT_PATH / "Pipfile").exists(): + self.run_command("pipenv install -dev") + + def _install_pip(self): + for requirements_file in self.CLONED_PROJECT_PATH.glob("*requirements.txt"): + self.run_command( + "pip install -r {}".format(requirements_file.absolute())) + + def _install_setup_py(self): + if (self.CLONED_PROJECT_PATH / "setup.py").exists(): + self.run_command("python setup.py install") + @property @lru_cache() def VENV_PATH(self): From f7ea88ef4fda1d65246b13142166637413ffe4c7 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 16:27:48 -0800 Subject: [PATCH 22/45] handle minetype == None --- langserver/fs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/langserver/fs.py b/langserver/fs.py index e507797..30fcecf 100644 --- a/langserver/fs.py +++ b/langserver/fs.py @@ -212,7 +212,8 @@ def _walk(self, top: str): if os.path.isdir(e): dirs.append(os.path.relpath(e, self.root)) else: - if mimetypes.guess_type(e)[0].startswith("text/"): + file_type = mimetypes.guess_type(e)[0] + if file_type and file_type.startswith("text/"): files.append(os.path.relpath(e, self.root)) yield from files for d in dirs: From 6a1b2860ab7e619086e4633581115ad25bfa2138 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 16:32:32 -0800 Subject: [PATCH 23/45] setup flask tests --- Makefile | 1 + test/test_flask.py | 1238 ++++++++++++++++++++++---------------------- 2 files changed, 616 insertions(+), 623 deletions(-) diff --git a/Makefile b/Makefile index 5cf6f4d..f667b2c 100644 --- a/Makefile +++ b/Makefile @@ -9,4 +9,5 @@ lint: test: # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_flask.py -vv cd ./test && pipenv run pytest test_fizzbuzz.py -vv diff --git a/test/test_flask.py b/test/test_flask.py index 6e5559e..87a6cbe 100644 --- a/test/test_flask.py +++ b/test/test_flask.py @@ -1,739 +1,731 @@ from .harness import Harness import uuid +import pytest + + +@pytest.fixture() +def flask_workspace(): + flask_workspace = Harness("repos/flask") + flask_workspace.initialize( + "git://github.com/pallets/flask?" + str(uuid.uuid4())) + yield flask_workspace + flask_workspace.exit() + + +class TestFlaskWorkspace: + + def test_x_packages(self, flask_workspace): + result = flask_workspace.x_packages() + assert result + result = result[0] + assert "package" in result and result["package"] == {'name': 'flask'} + assert "dependencies" in result + dep_names = {d["attributes"]["name"] for d in result["dependencies"]} + assert dep_names == { + 'blueprintapp', + 'hello', + 'minitwit', + 'werkzeug', + 'setuptools', + 'pytest', + 'itsdangerous', + 'cpython', + 'click', + 'yourapplication', + 'flaskr', + 'blueprintexample', + 'jinja2', + 'urllib2', + 'simple_page', + 'pkg_resources' + } + def test_local_hover(self, flask_workspace): + desired_result = { + 'contents': [ + { + 'language': 'python', + 'value': 'def find_best_app(param script_info, param module)' + }, + 'Given a module instance this tries to find the best possible\n' + 'application in the module or raises an exception.' + ] + } + # hover over the definition + result1 = flask_workspace.hover("/flask/cli.py", 34, 4) + assert result1 == desired_result + # hover over a usage + result2 = flask_workspace.hover("/flask/cli.py", 215, 15) + assert result2 == desired_result + + def test_cross_module_hover(self, flask_workspace): + result = flask_workspace.hover("/flask/app.py", 220, 12) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'class ConfigAttribute(param name, param ' + 'get_converter=None)' + }, + 'Makes an attribute forward to the config' + ] + } -flask_workspace = Harness("repos/flask") -flask_workspace.initialize( - "git://github.com/pallets/flask?" + str(uuid.uuid4())) - - -def test_x_packages(): - result = flask_workspace.x_packages() - assert result - result = result[0] - assert "package" in result and result["package"] == {'name': 'flask'} - assert "dependencies" in result - dep_names = {d["attributes"]["name"] for d in result["dependencies"]} - assert dep_names == { - 'blueprintapp', - 'hello', - 'minitwit', - 'werkzeug', - 'setuptools', - 'pytest', - 'itsdangerous', - 'cpython', - 'click', - 'yourapplication', - 'flaskr', - 'blueprintexample', - 'jinja2', - 'urllib2', - 'simple_page', - 'pkg_resources' - } - - -def test_local_hover(): - desired_result = { - 'contents': [ - { - 'language': 'python', - 'value': 'def find_best_app(param script_info, param module)' - }, - 'Given a module instance this tries to find the best possible\n' - 'application in the module or raises an exception.' - ] - } - # hover over the definition - result1 = flask_workspace.hover("/flask/cli.py", 34, 4) - assert result1 == desired_result - # hover over a usage - result2 = flask_workspace.hover("/flask/cli.py", 215, 15) - assert result2 == desired_result - + def test_cross_package_hover(self, flask_workspace): + result = flask_workspace.hover("/flask/__init__.py", 44, 15) + assert "contents" in result + assert result["contents"] + assert result["contents"][0] == { + 'language': 'python', + 'value': 'def jsonify(param *args, param **kwargs)' + } -def test_cross_module_hover(): - result = flask_workspace.hover("/flask/app.py", 220, 12) - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'class ConfigAttribute(param name, param ' - 'get_converter=None)' + # TODO(aaron): the actual definition results have duplicates for some + # reason ... maybe the TestFileSystem? + def test_local_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/cli.py", 215, 15) + symbol = { + 'location': { + 'range': { + 'end': { + 'character': 17, + 'line': 34 + }, + 'start': { + 'character': 4, + 'line': 34 + } + }, + 'uri': 'file:///flask/cli.py' }, - 'Makes an attribute forward to the config' - ] - } - - -def test_cross_package_hover(): - result = flask_workspace.hover("/flask/__init__.py", 44, 15) - assert "contents" in result - assert result["contents"] - assert result["contents"][0] == { - 'language': 'python', - 'value': 'def jsonify(param *args, param **kwargs)' - } - - -# TODO(aaron): the actual definition results have duplicates for some -# reason ... maybe the TestFileSystem? -def test_local_definition(): - result = flask_workspace.definition("/flask/cli.py", 215, 15) - symbol = { - 'location': { - 'range': { - 'end': { - 'character': 17, - 'line': 34 + 'symbol': { + 'container': 'flask.cli', + 'file': 'cli.py', + 'kind': 'def', + 'name': 'find_best_app', + 'package': { + 'name': 'flask' }, - 'start': { + 'position': { 'character': 4, 'line': 34 } - }, - 'uri': 'file:///flask/cli.py' - }, - 'symbol': { - 'container': 'flask.cli', - 'file': 'cli.py', - 'kind': 'def', - 'name': 'find_best_app', - 'package': { - 'name': 'flask' - }, - 'position': { - 'character': 4, - 'line': 34 } } - } - assert symbol in result - + assert symbol in result -def test_cross_module_definition(): - result = flask_workspace.definition("/flask/app.py", 220, 12) - symbol = { - 'location': { - 'range': { - 'end': { - 'character': 43, - 'line': 26 + def test_cross_module_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/app.py", 220, 12) + symbol = { + 'location': { + 'range': { + 'end': { + 'character': 43, + 'line': 26 + }, + 'start': { + 'character': 28, + 'line': 26 + } }, - 'start': { - 'character': 28, - 'line': 26 - } - }, - 'uri': 'file:///flask/app.py' - }, - 'symbol': { - 'container': 'flask.app', - 'file': 'app.py', - 'kind': 'class', - 'name': 'ConfigAttribute', - 'package': { - 'name': 'flask' + 'uri': 'file:///flask/app.py' }, - 'position': { - 'character': 28, - 'line': 26 - } - } - } - assert symbol in result - - -def test_cross_package_definition(): - result = flask_workspace.definition("/flask/__init__.py", 44, 15) - symbol = { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'jsonify', - 'container': 'flask.json', - 'kind': 'def', - 'file': '__init__.py', - 'position': { - 'line': 202, - 'character': 4 - } - }, - 'location': { - 'uri': 'file:///flask/json/__init__.py', - 'range': { - 'start': { - 'line': 202, - 'character': 4 + 'symbol': { + 'container': 'flask.app', + 'file': 'app.py', + 'kind': 'class', + 'name': 'ConfigAttribute', + 'package': { + 'name': 'flask' }, - 'end': { - 'line': 202, - 'character': 11 + 'position': { + 'character': 28, + 'line': 26 } } } - } - assert symbol in result + assert symbol in result - -def test_local_package_import_definition(): - result = flask_workspace.definition("/flask/__init__.py", 44, 10) - assert result == [ - { + def test_cross_package_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/__init__.py", 44, 15) + symbol = { 'symbol': { 'package': { 'name': 'flask' }, - 'name': 'json', + 'name': 'jsonify', 'container': 'flask.json', - 'kind': 'module', + 'kind': 'def', 'file': '__init__.py', 'position': { - 'line': 0, - 'character': 0 + 'line': 202, + 'character': 4 } }, 'location': { 'uri': 'file:///flask/json/__init__.py', 'range': { 'start': { - 'line': 0, - 'character': 0 + 'line': 202, + 'character': 4 }, 'end': { - 'line': 0, - 'character': 4 + 'line': 202, + 'character': 11 } } } - }, - { - 'symbol': { - 'package': { - 'name': 'flask' + } + assert symbol in result + + def test_local_package_import_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/__init__.py", 44, 10) + assert result == [ + { + 'symbol': { + 'package': { + 'name': 'flask' + }, + 'name': 'json', + 'container': 'flask.json', + 'kind': 'module', + 'file': '__init__.py', + 'position': { + 'line': 0, + 'character': 0 + } }, - 'name': 'json', - 'container': 'flask', - 'kind': 'module', - 'file': '__init__.py', - 'position': { - 'line': 40, - 'character': 14 + 'location': { + 'uri': 'file:///flask/json/__init__.py', + 'range': { + 'start': { + 'line': 0, + 'character': 0 + }, + 'end': { + 'line': 0, + 'character': 4 + } + } } }, - 'location': { - 'uri': 'file:///flask/__init__.py', - 'range': { - 'start': { - 'line': 40, - 'character': 14 + { + 'symbol': { + 'package': { + 'name': 'flask' }, - 'end': { + 'name': 'json', + 'container': 'flask', + 'kind': 'module', + 'file': '__init__.py', + 'position': { 'line': 40, - 'character': 18 + 'character': 14 + } + }, + 'location': { + 'uri': 'file:///flask/__init__.py', + 'range': { + 'start': { + 'line': 40, + 'character': 14 + }, + 'end': { + 'line': 40, + 'character': 18 + } } } - } - }, - ] - - -def test_cross_repo_hover(): - result = flask_workspace.hover("/flask/app.py", 295, 20) - assert result == { - 'contents': [ - { - 'language': 'python', - 'value': 'class ImmutableDict(param type(self))' }, - 'An immutable :class:`dict`.\n\n.. versionadded:: 0.5' ] - } + def test_cross_repo_hover(self, flask_workspace): + result = flask_workspace.hover("/flask/app.py", 295, 20) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'class ImmutableDict(param type(self))' + }, + 'An immutable :class:`dict`.\n\n.. versionadded:: 0.5' + ] + } -def test_cross_repo_definition(): - result = flask_workspace.definition("/flask/app.py", 295, 20) - # TODO(aaron): should we return a symbol descriptor with the local result? - # Might screw up xrefs - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'werkzeug' + def test_cross_repo_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/app.py", 295, 20) + # TODO(aaron): should we return a symbol descriptor with the local result? + # Might screw up xrefs + assert result == [ + { + 'symbol': { + 'package': { + 'name': 'werkzeug' + }, + 'name': 'ImmutableDict', + 'container': 'werkzeug.datastructures', + 'kind': 'class', + 'file': 'datastructures.py', + 'position': { + 'line': 1541, + 'character': 6 + } }, - 'name': 'ImmutableDict', - 'container': 'werkzeug.datastructures', - 'kind': 'class', - 'file': 'datastructures.py', - 'position': { - 'line': 1541, - 'character': 6 - } + 'location': None }, - 'location': None - }, - { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'ImmutableDict', - 'container': 'flask.app', - 'kind': 'class', - 'file': 'app.py', - 'position': { - 'line': 18, - 'character': 36 - } - }, - 'location': { - 'uri': 'file:///flask/app.py', - 'range': { - 'start': { - 'line': 18, - 'character': 36 + { + 'symbol': { + 'package': { + 'name': 'flask' }, - 'end': { + 'name': 'ImmutableDict', + 'container': 'flask.app', + 'kind': 'class', + 'file': 'app.py', + 'position': { 'line': 18, - 'character': 49 + 'character': 36 + } + }, + 'location': { + 'uri': 'file:///flask/app.py', + 'range': { + 'start': { + 'line': 18, + 'character': 36 + }, + 'end': { + 'line': 18, + 'character': 49 + } } } - } - }, - ] - + }, + ] -def test_cross_repo_import_definition(): - result = flask_workspace.definition("/flask/__init__.py", 18, 19) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'markupsafe' + def test_cross_repo_import_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/__init__.py", 18, 19) + assert result == [ + { + 'symbol': { + 'package': { + 'name': 'markupsafe' + }, + 'name': 'Markup', + 'container': 'markupsafe', + 'kind': 'class', + 'file': '__init__.py', + 'position': { + 'line': 25, + 'character': 6 + } }, - 'name': 'Markup', - 'container': 'markupsafe', - 'kind': 'class', - 'file': '__init__.py', - 'position': { - 'line': 25, - 'character': 6 - } + 'location': None }, - 'location': None - }, - { - 'symbol': { - 'package': { - 'name': 'jinja2' + { + 'symbol': { + 'package': { + 'name': 'jinja2' + }, + 'name': 'Markup', + 'container': 'jinja2', + 'kind': 'class', + 'file': '__init__.py', + 'position': { + 'line': 55, + 'character': 25 + } }, - 'name': 'Markup', - 'container': 'jinja2', - 'kind': 'class', - 'file': '__init__.py', - 'position': { - 'line': 55, - 'character': 25 - } + 'location': None }, - 'location': None - }, - ] - + ] -def test_stdlib_hover(): - result = flask_workspace.hover("/flask/app.py", 306, 48) - assert result == { - 'contents': [ + def test_stdlib_hover(self, flask_workspace): + result = flask_workspace.hover("/flask/app.py", 306, 48) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'class timedelta(param type(self))'}, + 'Represent the difference between two datetime objects.\n\n' + 'Supported operators:\n\n' + '- add, subtract timedelta\n' + '- unary plus, minus, abs\n' + '- compare to timedelta\n' + '- multiply, divide by int\n\n' + 'In addition, datetime supports subtraction of two datetime objects\n' + 'returning a timedelta, and addition or subtraction of a datetime\n' + 'and a timedelta giving a datetime.\n\n' + 'Representation: (days, seconds, microseconds). Why? Because I\n' + 'felt like it.']} + + def test_stdlib_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/app.py", 306, 48) + assert result == [ { - 'language': 'python', - 'value': 'class timedelta(param type(self))'}, - 'Represent the difference between two datetime objects.\n\n' - 'Supported operators:\n\n' - '- add, subtract timedelta\n' - '- unary plus, minus, abs\n' - '- compare to timedelta\n' - '- multiply, divide by int\n\n' - 'In addition, datetime supports subtraction of two datetime objects\n' - 'returning a timedelta, and addition or subtraction of a datetime\n' - 'and a timedelta giving a datetime.\n\n' - 'Representation: (days, seconds, microseconds). Why? Because I\n' - 'felt like it.']} - - -def test_stdlib_definition(): - result = flask_workspace.definition("/flask/app.py", 306, 48) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'cpython' + 'symbol': { + 'package': { + 'name': 'cpython' + }, + 'name': 'timedelta', + 'container': 'datetime', + 'kind': 'class', + 'path': 'Lib/datetime.py', + 'file': 'datetime.py', + 'position': { + 'line': 335, + 'character': 6 + } }, - 'name': 'timedelta', - 'container': 'datetime', - 'kind': 'class', - 'path': 'Lib/datetime.py', - 'file': 'datetime.py', - 'position': { - 'line': 335, - 'character': 6 - } + 'location': None }, - 'location': None - }, - { - 'symbol': { - 'package': { - 'name': 'flask' + { + 'symbol': { + 'package': { + 'name': 'flask' + }, + 'name': 'timedelta', + 'container': 'flask.app', + 'kind': 'class', + 'file': 'app.py', + 'position': { + 'line': 13, + 'character': 21 + } }, - 'name': 'timedelta', - 'container': 'flask.app', - 'kind': 'class', - 'file': 'app.py', - 'position': { - 'line': 13, - 'character': 21 + 'location': { + 'uri': 'file:///flask/app.py', + 'range': { + 'start': { + 'line': 13, + 'character': 21 + }, + 'end': { + 'line': 13, + 'character': 30 + } + } } }, - 'location': { - 'uri': 'file:///flask/app.py', + ] + + def test_local_references(self, flask_workspace): + result = flask_workspace.references("/flask/cli.py", 34, 4) + assert result == [ + { + 'uri': 'file:///flask/cli.py', 'range': { 'start': { - 'line': 13, - 'character': 21 + 'line': 215, + 'character': 15 }, 'end': { - 'line': 13, - 'character': 30 + 'line': 215, + 'character': 28 } } - } - }, - ] - - -def test_local_references(): - result = flask_workspace.references("/flask/cli.py", 34, 4) - assert result == [ - { - 'uri': 'file:///flask/cli.py', - 'range': { - 'start': { - 'line': 215, - 'character': 15 - }, - 'end': { - 'line': 215, - 'character': 28 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 46, - 'character': 11 - }, - 'end': { - 'line': 46, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 51, - 'character': 11 - }, - 'end': { - 'line': 51, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 56, - 'character': 11 - }, - 'end': { - 'line': 56, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 63, - 'character': 22 - }, - 'end': { - 'line': 63, - 'character': 35 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 64, - 'character': 11 - }, - 'end': { - 'line': 64, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 71, - 'character': 22 - }, - 'end': { - 'line': 71, - 'character': 35 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 46, + 'character': 11 + }, + 'end': { + 'line': 46, + 'character': 24 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 72, - 'character': 11 - }, - 'end': { - 'line': 72, - 'character': 24 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 51, + 'character': 11 + }, + 'end': { + 'line': 51, + 'character': 24 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 79, - 'character': 22 - }, - 'end': { - 'line': 79, - 'character': 35 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 56, + 'character': 11 + }, + 'end': { + 'line': 56, + 'character': 24 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 80, - 'character': 11 - }, - 'end': { - 'line': 80, - 'character': 24 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 63, + 'character': 22 + }, + 'end': { + 'line': 63, + 'character': 35 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 87, - 'character': 22 - }, - 'end': { - 'line': 87, - 'character': 35 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 64, + 'character': 11 + }, + 'end': { + 'line': 64, + 'character': 24 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 88, - 'character': 11 - }, - 'end': { - 'line': 88, - 'character': 24 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 71, + 'character': 22 + }, + 'end': { + 'line': 71, + 'character': 35 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 97, - 'character': 11 - }, - 'end': { - 'line': 97, - 'character': 24 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 72, + 'character': 11 + }, + 'end': { + 'line': 72, + 'character': 24 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 106, - 'character': 11 - }, - 'end': { - 'line': 106, - 'character': 24 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 79, + 'character': 22 + }, + 'end': { + 'line': 79, + 'character': 35 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 111, - 'character': 34 - }, - 'end': { - 'line': 111, - 'character': 47 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 80, + 'character': 11 + }, + 'end': { + 'line': 80, + 'character': 24 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 117, - 'character': 34 - }, - 'end': { - 'line': 117, - 'character': 47 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 87, + 'character': 22 + }, + 'end': { + 'line': 87, + 'character': 35 + } } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 124, - 'character': 34 - }, - 'end': { - 'line': 124, - 'character': 47 + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 88, + 'character': 11 + }, + 'end': { + 'line': 88, + 'character': 24 + } } - } - } - ] - - -def test_x_references(): - result = flask_workspace.x_references( - "werkzeug.datastructures", "ImmutableDict") - assert result == [ - { - 'reference': { - 'uri': 'file:///flask/app.py', + }, + { + 'uri': 'file:///tests/test_cli.py', 'range': { 'start': { - 'line': 18, - 'character': 0 + 'line': 97, + 'character': 11 }, 'end': { - 'line': 18, - 'character': 13 + 'line': 97, + 'character': 24 } } }, - 'symbol': { - 'container': 'werkzeug.datastructures', - 'name': 'ImmutableDict' - } - }, - { - 'reference': { - 'uri': 'file:///flask/app.py', + { + 'uri': 'file:///tests/test_cli.py', 'range': { 'start': { - 'line': 295, - 'character': 20 + 'line': 106, + 'character': 11 }, 'end': { - 'line': 295, - 'character': 33 + 'line': 106, + 'character': 24 } } }, - 'symbol': { - 'container': 'werkzeug.datastructures', - 'name': 'ImmutableDict' - } - }, - { - 'reference': { - 'uri': 'file:///flask/app.py', + { + 'uri': 'file:///tests/test_cli.py', 'range': { 'start': { - 'line': 300, - 'character': 21 + 'line': 111, + 'character': 34 }, 'end': { - 'line': 300, + 'line': 111, + 'character': 47 + } + } + }, + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 117, 'character': 34 + }, + 'end': { + 'line': 117, + 'character': 47 } } }, - 'symbol': { - 'container': 'werkzeug.datastructures', - 'name': 'ImmutableDict' + { + 'uri': 'file:///tests/test_cli.py', + 'range': { + 'start': { + 'line': 124, + 'character': 34 + }, + 'end': { + 'line': 124, + 'character': 47 + } + } } - } - ] - + ] -def test_definition_of_definition(): - result = flask_workspace.definition("/flask/blueprints.py", 142, 8) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'flask' + def test_x_references(self, flask_workspace): + result = flask_workspace.x_references( + "werkzeug.datastructures", "ImmutableDict") + assert result == [ + { + 'reference': { + 'uri': 'file:///flask/app.py', + 'range': { + 'start': { + 'line': 18, + 'character': 0 + }, + 'end': { + 'line': 18, + 'character': 13 + } + } }, - 'name': 'record_once', - 'container': 'flask.blueprints', - 'kind': 'def', - 'file': 'blueprints.py', - 'position': { - 'line': 142, - 'character': 8 + 'symbol': { + 'container': 'werkzeug.datastructures', + 'name': 'ImmutableDict' } }, - 'location': { - 'uri': 'file:///flask/blueprints.py', - 'range': { - 'start': { - 'line': 142, - 'character': 8 + { + 'reference': { + 'uri': 'file:///flask/app.py', + 'range': { + 'start': { + 'line': 295, + 'character': 20 + }, + 'end': { + 'line': 295, + 'character': 33 + } + } + }, + 'symbol': { + 'container': 'werkzeug.datastructures', + 'name': 'ImmutableDict' + } + }, + { + 'reference': { + 'uri': 'file:///flask/app.py', + 'range': { + 'start': { + 'line': 300, + 'character': 21 + }, + 'end': { + 'line': 300, + 'character': 34 + } + } + }, + 'symbol': { + 'container': 'werkzeug.datastructures', + 'name': 'ImmutableDict' + } + } + ] + + def test_definition_of_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/blueprints.py", 142, 8) + assert result == [ + { + 'symbol': { + 'package': { + 'name': 'flask' }, - 'end': { + 'name': 'record_once', + 'container': 'flask.blueprints', + 'kind': 'def', + 'file': 'blueprints.py', + 'position': { 'line': 142, - 'character': 19 + 'character': 8 + } + }, + 'location': { + 'uri': 'file:///flask/blueprints.py', + 'range': { + 'start': { + 'line': 142, + 'character': 8 + }, + 'end': { + 'line': 142, + 'character': 19 + } } } } - } - ] + ] From 2f1079b7095ceac80f63cdd2d3a2eadfe0ab7fb5 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 16:36:19 -0800 Subject: [PATCH 24/45] comment out all non hover tests --- test/test_flask.py | 1268 ++++++++++++++++++++++---------------------- 1 file changed, 634 insertions(+), 634 deletions(-) diff --git a/test/test_flask.py b/test/test_flask.py index 87a6cbe..935d350 100644 --- a/test/test_flask.py +++ b/test/test_flask.py @@ -14,31 +14,31 @@ def flask_workspace(): class TestFlaskWorkspace: - def test_x_packages(self, flask_workspace): - result = flask_workspace.x_packages() - assert result - result = result[0] - assert "package" in result and result["package"] == {'name': 'flask'} - assert "dependencies" in result - dep_names = {d["attributes"]["name"] for d in result["dependencies"]} - assert dep_names == { - 'blueprintapp', - 'hello', - 'minitwit', - 'werkzeug', - 'setuptools', - 'pytest', - 'itsdangerous', - 'cpython', - 'click', - 'yourapplication', - 'flaskr', - 'blueprintexample', - 'jinja2', - 'urllib2', - 'simple_page', - 'pkg_resources' - } + # def test_x_packages(self, flask_workspace): + # result = flask_workspace.x_packages() + # assert result + # result = result[0] + # assert "package" in result and result["package"] == {'name': 'flask'} + # assert "dependencies" in result + # dep_names = {d["attributes"]["name"] for d in result["dependencies"]} + # assert dep_names == { + # 'blueprintapp', + # 'hello', + # 'minitwit', + # 'werkzeug', + # 'setuptools', + # 'pytest', + # 'itsdangerous', + # 'cpython', + # 'click', + # 'yourapplication', + # 'flaskr', + # 'blueprintexample', + # 'jinja2', + # 'urllib2', + # 'simple_page', + # 'pkg_resources' + # } def test_local_hover(self, flask_workspace): desired_result = { @@ -80,164 +80,164 @@ def test_cross_package_hover(self, flask_workspace): 'value': 'def jsonify(param *args, param **kwargs)' } - # TODO(aaron): the actual definition results have duplicates for some - # reason ... maybe the TestFileSystem? - def test_local_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/cli.py", 215, 15) - symbol = { - 'location': { - 'range': { - 'end': { - 'character': 17, - 'line': 34 - }, - 'start': { - 'character': 4, - 'line': 34 - } - }, - 'uri': 'file:///flask/cli.py' - }, - 'symbol': { - 'container': 'flask.cli', - 'file': 'cli.py', - 'kind': 'def', - 'name': 'find_best_app', - 'package': { - 'name': 'flask' - }, - 'position': { - 'character': 4, - 'line': 34 - } - } - } - assert symbol in result + # # TODO(aaron): the actual definition results have duplicates for some + # # reason ... maybe the TestFileSystem? + # def test_local_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/cli.py", 215, 15) + # symbol = { + # 'location': { + # 'range': { + # 'end': { + # 'character': 17, + # 'line': 34 + # }, + # 'start': { + # 'character': 4, + # 'line': 34 + # } + # }, + # 'uri': 'file:///flask/cli.py' + # }, + # 'symbol': { + # 'container': 'flask.cli', + # 'file': 'cli.py', + # 'kind': 'def', + # 'name': 'find_best_app', + # 'package': { + # 'name': 'flask' + # }, + # 'position': { + # 'character': 4, + # 'line': 34 + # } + # } + # } + # assert symbol in result - def test_cross_module_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/app.py", 220, 12) - symbol = { - 'location': { - 'range': { - 'end': { - 'character': 43, - 'line': 26 - }, - 'start': { - 'character': 28, - 'line': 26 - } - }, - 'uri': 'file:///flask/app.py' - }, - 'symbol': { - 'container': 'flask.app', - 'file': 'app.py', - 'kind': 'class', - 'name': 'ConfigAttribute', - 'package': { - 'name': 'flask' - }, - 'position': { - 'character': 28, - 'line': 26 - } - } - } - assert symbol in result + # def test_cross_module_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/app.py", 220, 12) + # symbol = { + # 'location': { + # 'range': { + # 'end': { + # 'character': 43, + # 'line': 26 + # }, + # 'start': { + # 'character': 28, + # 'line': 26 + # } + # }, + # 'uri': 'file:///flask/app.py' + # }, + # 'symbol': { + # 'container': 'flask.app', + # 'file': 'app.py', + # 'kind': 'class', + # 'name': 'ConfigAttribute', + # 'package': { + # 'name': 'flask' + # }, + # 'position': { + # 'character': 28, + # 'line': 26 + # } + # } + # } + # assert symbol in result - def test_cross_package_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/__init__.py", 44, 15) - symbol = { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'jsonify', - 'container': 'flask.json', - 'kind': 'def', - 'file': '__init__.py', - 'position': { - 'line': 202, - 'character': 4 - } - }, - 'location': { - 'uri': 'file:///flask/json/__init__.py', - 'range': { - 'start': { - 'line': 202, - 'character': 4 - }, - 'end': { - 'line': 202, - 'character': 11 - } - } - } - } - assert symbol in result + # def test_cross_package_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/__init__.py", 44, 15) + # symbol = { + # 'symbol': { + # 'package': { + # 'name': 'flask' + # }, + # 'name': 'jsonify', + # 'container': 'flask.json', + # 'kind': 'def', + # 'file': '__init__.py', + # 'position': { + # 'line': 202, + # 'character': 4 + # } + # }, + # 'location': { + # 'uri': 'file:///flask/json/__init__.py', + # 'range': { + # 'start': { + # 'line': 202, + # 'character': 4 + # }, + # 'end': { + # 'line': 202, + # 'character': 11 + # } + # } + # } + # } + # assert symbol in result - def test_local_package_import_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/__init__.py", 44, 10) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'json', - 'container': 'flask.json', - 'kind': 'module', - 'file': '__init__.py', - 'position': { - 'line': 0, - 'character': 0 - } - }, - 'location': { - 'uri': 'file:///flask/json/__init__.py', - 'range': { - 'start': { - 'line': 0, - 'character': 0 - }, - 'end': { - 'line': 0, - 'character': 4 - } - } - } - }, - { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'json', - 'container': 'flask', - 'kind': 'module', - 'file': '__init__.py', - 'position': { - 'line': 40, - 'character': 14 - } - }, - 'location': { - 'uri': 'file:///flask/__init__.py', - 'range': { - 'start': { - 'line': 40, - 'character': 14 - }, - 'end': { - 'line': 40, - 'character': 18 - } - } - } - }, - ] + # def test_local_package_import_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/__init__.py", 44, 10) + # assert result == [ + # { + # 'symbol': { + # 'package': { + # 'name': 'flask' + # }, + # 'name': 'json', + # 'container': 'flask.json', + # 'kind': 'module', + # 'file': '__init__.py', + # 'position': { + # 'line': 0, + # 'character': 0 + # } + # }, + # 'location': { + # 'uri': 'file:///flask/json/__init__.py', + # 'range': { + # 'start': { + # 'line': 0, + # 'character': 0 + # }, + # 'end': { + # 'line': 0, + # 'character': 4 + # } + # } + # } + # }, + # { + # 'symbol': { + # 'package': { + # 'name': 'flask' + # }, + # 'name': 'json', + # 'container': 'flask', + # 'kind': 'module', + # 'file': '__init__.py', + # 'position': { + # 'line': 40, + # 'character': 14 + # } + # }, + # 'location': { + # 'uri': 'file:///flask/__init__.py', + # 'range': { + # 'start': { + # 'line': 40, + # 'character': 14 + # }, + # 'end': { + # 'line': 40, + # 'character': 18 + # } + # } + # } + # }, + # ] def test_cross_repo_hover(self, flask_workspace): result = flask_workspace.hover("/flask/app.py", 295, 20) @@ -251,93 +251,93 @@ def test_cross_repo_hover(self, flask_workspace): ] } - def test_cross_repo_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/app.py", 295, 20) - # TODO(aaron): should we return a symbol descriptor with the local result? - # Might screw up xrefs - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'werkzeug' - }, - 'name': 'ImmutableDict', - 'container': 'werkzeug.datastructures', - 'kind': 'class', - 'file': 'datastructures.py', - 'position': { - 'line': 1541, - 'character': 6 - } - }, - 'location': None - }, - { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'ImmutableDict', - 'container': 'flask.app', - 'kind': 'class', - 'file': 'app.py', - 'position': { - 'line': 18, - 'character': 36 - } - }, - 'location': { - 'uri': 'file:///flask/app.py', - 'range': { - 'start': { - 'line': 18, - 'character': 36 - }, - 'end': { - 'line': 18, - 'character': 49 - } - } - } - }, - ] + # def test_cross_repo_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/app.py", 295, 20) + # # TODO(aaron): should we return a symbol descriptor with the local result? + # # Might screw up xrefs + # assert result == [ + # { + # 'symbol': { + # 'package': { + # 'name': 'werkzeug' + # }, + # 'name': 'ImmutableDict', + # 'container': 'werkzeug.datastructures', + # 'kind': 'class', + # 'file': 'datastructures.py', + # 'position': { + # 'line': 1541, + # 'character': 6 + # } + # }, + # 'location': None + # }, + # { + # 'symbol': { + # 'package': { + # 'name': 'flask' + # }, + # 'name': 'ImmutableDict', + # 'container': 'flask.app', + # 'kind': 'class', + # 'file': 'app.py', + # 'position': { + # 'line': 18, + # 'character': 36 + # } + # }, + # 'location': { + # 'uri': 'file:///flask/app.py', + # 'range': { + # 'start': { + # 'line': 18, + # 'character': 36 + # }, + # 'end': { + # 'line': 18, + # 'character': 49 + # } + # } + # } + # }, + # ] - def test_cross_repo_import_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/__init__.py", 18, 19) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'markupsafe' - }, - 'name': 'Markup', - 'container': 'markupsafe', - 'kind': 'class', - 'file': '__init__.py', - 'position': { - 'line': 25, - 'character': 6 - } - }, - 'location': None - }, - { - 'symbol': { - 'package': { - 'name': 'jinja2' - }, - 'name': 'Markup', - 'container': 'jinja2', - 'kind': 'class', - 'file': '__init__.py', - 'position': { - 'line': 55, - 'character': 25 - } - }, - 'location': None - }, - ] + # def test_cross_repo_import_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/__init__.py", 18, 19) + # assert result == [ + # { + # 'symbol': { + # 'package': { + # 'name': 'markupsafe' + # }, + # 'name': 'Markup', + # 'container': 'markupsafe', + # 'kind': 'class', + # 'file': '__init__.py', + # 'position': { + # 'line': 25, + # 'character': 6 + # } + # }, + # 'location': None + # }, + # { + # 'symbol': { + # 'package': { + # 'name': 'jinja2' + # }, + # 'name': 'Markup', + # 'container': 'jinja2', + # 'kind': 'class', + # 'file': '__init__.py', + # 'position': { + # 'line': 55, + # 'character': 25 + # } + # }, + # 'location': None + # }, + # ] def test_stdlib_hover(self, flask_workspace): result = flask_workspace.hover("/flask/app.py", 306, 48) @@ -358,374 +358,374 @@ def test_stdlib_hover(self, flask_workspace): 'Representation: (days, seconds, microseconds). Why? Because I\n' 'felt like it.']} - def test_stdlib_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/app.py", 306, 48) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'cpython' - }, - 'name': 'timedelta', - 'container': 'datetime', - 'kind': 'class', - 'path': 'Lib/datetime.py', - 'file': 'datetime.py', - 'position': { - 'line': 335, - 'character': 6 - } - }, - 'location': None - }, - { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'timedelta', - 'container': 'flask.app', - 'kind': 'class', - 'file': 'app.py', - 'position': { - 'line': 13, - 'character': 21 - } - }, - 'location': { - 'uri': 'file:///flask/app.py', - 'range': { - 'start': { - 'line': 13, - 'character': 21 - }, - 'end': { - 'line': 13, - 'character': 30 - } - } - } - }, - ] + # def test_stdlib_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/app.py", 306, 48) + # assert result == [ + # { + # 'symbol': { + # 'package': { + # 'name': 'cpython' + # }, + # 'name': 'timedelta', + # 'container': 'datetime', + # 'kind': 'class', + # 'path': 'Lib/datetime.py', + # 'file': 'datetime.py', + # 'position': { + # 'line': 335, + # 'character': 6 + # } + # }, + # 'location': None + # }, + # { + # 'symbol': { + # 'package': { + # 'name': 'flask' + # }, + # 'name': 'timedelta', + # 'container': 'flask.app', + # 'kind': 'class', + # 'file': 'app.py', + # 'position': { + # 'line': 13, + # 'character': 21 + # } + # }, + # 'location': { + # 'uri': 'file:///flask/app.py', + # 'range': { + # 'start': { + # 'line': 13, + # 'character': 21 + # }, + # 'end': { + # 'line': 13, + # 'character': 30 + # } + # } + # } + # }, + # ] - def test_local_references(self, flask_workspace): - result = flask_workspace.references("/flask/cli.py", 34, 4) - assert result == [ - { - 'uri': 'file:///flask/cli.py', - 'range': { - 'start': { - 'line': 215, - 'character': 15 - }, - 'end': { - 'line': 215, - 'character': 28 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 46, - 'character': 11 - }, - 'end': { - 'line': 46, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 51, - 'character': 11 - }, - 'end': { - 'line': 51, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 56, - 'character': 11 - }, - 'end': { - 'line': 56, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 63, - 'character': 22 - }, - 'end': { - 'line': 63, - 'character': 35 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 64, - 'character': 11 - }, - 'end': { - 'line': 64, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 71, - 'character': 22 - }, - 'end': { - 'line': 71, - 'character': 35 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 72, - 'character': 11 - }, - 'end': { - 'line': 72, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 79, - 'character': 22 - }, - 'end': { - 'line': 79, - 'character': 35 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 80, - 'character': 11 - }, - 'end': { - 'line': 80, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 87, - 'character': 22 - }, - 'end': { - 'line': 87, - 'character': 35 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 88, - 'character': 11 - }, - 'end': { - 'line': 88, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 97, - 'character': 11 - }, - 'end': { - 'line': 97, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 106, - 'character': 11 - }, - 'end': { - 'line': 106, - 'character': 24 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 111, - 'character': 34 - }, - 'end': { - 'line': 111, - 'character': 47 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 117, - 'character': 34 - }, - 'end': { - 'line': 117, - 'character': 47 - } - } - }, - { - 'uri': 'file:///tests/test_cli.py', - 'range': { - 'start': { - 'line': 124, - 'character': 34 - }, - 'end': { - 'line': 124, - 'character': 47 - } - } - } - ] + # def test_local_references(self, flask_workspace): + # result = flask_workspace.references("/flask/cli.py", 34, 4) + # assert result == [ + # { + # 'uri': 'file:///flask/cli.py', + # 'range': { + # 'start': { + # 'line': 215, + # 'character': 15 + # }, + # 'end': { + # 'line': 215, + # 'character': 28 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 46, + # 'character': 11 + # }, + # 'end': { + # 'line': 46, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 51, + # 'character': 11 + # }, + # 'end': { + # 'line': 51, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 56, + # 'character': 11 + # }, + # 'end': { + # 'line': 56, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 63, + # 'character': 22 + # }, + # 'end': { + # 'line': 63, + # 'character': 35 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 64, + # 'character': 11 + # }, + # 'end': { + # 'line': 64, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 71, + # 'character': 22 + # }, + # 'end': { + # 'line': 71, + # 'character': 35 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 72, + # 'character': 11 + # }, + # 'end': { + # 'line': 72, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 79, + # 'character': 22 + # }, + # 'end': { + # 'line': 79, + # 'character': 35 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 80, + # 'character': 11 + # }, + # 'end': { + # 'line': 80, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 87, + # 'character': 22 + # }, + # 'end': { + # 'line': 87, + # 'character': 35 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 88, + # 'character': 11 + # }, + # 'end': { + # 'line': 88, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 97, + # 'character': 11 + # }, + # 'end': { + # 'line': 97, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 106, + # 'character': 11 + # }, + # 'end': { + # 'line': 106, + # 'character': 24 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 111, + # 'character': 34 + # }, + # 'end': { + # 'line': 111, + # 'character': 47 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 117, + # 'character': 34 + # }, + # 'end': { + # 'line': 117, + # 'character': 47 + # } + # } + # }, + # { + # 'uri': 'file:///tests/test_cli.py', + # 'range': { + # 'start': { + # 'line': 124, + # 'character': 34 + # }, + # 'end': { + # 'line': 124, + # 'character': 47 + # } + # } + # } + # ] - def test_x_references(self, flask_workspace): - result = flask_workspace.x_references( - "werkzeug.datastructures", "ImmutableDict") - assert result == [ - { - 'reference': { - 'uri': 'file:///flask/app.py', - 'range': { - 'start': { - 'line': 18, - 'character': 0 - }, - 'end': { - 'line': 18, - 'character': 13 - } - } - }, - 'symbol': { - 'container': 'werkzeug.datastructures', - 'name': 'ImmutableDict' - } - }, - { - 'reference': { - 'uri': 'file:///flask/app.py', - 'range': { - 'start': { - 'line': 295, - 'character': 20 - }, - 'end': { - 'line': 295, - 'character': 33 - } - } - }, - 'symbol': { - 'container': 'werkzeug.datastructures', - 'name': 'ImmutableDict' - } - }, - { - 'reference': { - 'uri': 'file:///flask/app.py', - 'range': { - 'start': { - 'line': 300, - 'character': 21 - }, - 'end': { - 'line': 300, - 'character': 34 - } - } - }, - 'symbol': { - 'container': 'werkzeug.datastructures', - 'name': 'ImmutableDict' - } - } - ] + # def test_x_references(self, flask_workspace): + # result = flask_workspace.x_references( + # "werkzeug.datastructures", "ImmutableDict") + # assert result == [ + # { + # 'reference': { + # 'uri': 'file:///flask/app.py', + # 'range': { + # 'start': { + # 'line': 18, + # 'character': 0 + # }, + # 'end': { + # 'line': 18, + # 'character': 13 + # } + # } + # }, + # 'symbol': { + # 'container': 'werkzeug.datastructures', + # 'name': 'ImmutableDict' + # } + # }, + # { + # 'reference': { + # 'uri': 'file:///flask/app.py', + # 'range': { + # 'start': { + # 'line': 295, + # 'character': 20 + # }, + # 'end': { + # 'line': 295, + # 'character': 33 + # } + # } + # }, + # 'symbol': { + # 'container': 'werkzeug.datastructures', + # 'name': 'ImmutableDict' + # } + # }, + # { + # 'reference': { + # 'uri': 'file:///flask/app.py', + # 'range': { + # 'start': { + # 'line': 300, + # 'character': 21 + # }, + # 'end': { + # 'line': 300, + # 'character': 34 + # } + # } + # }, + # 'symbol': { + # 'container': 'werkzeug.datastructures', + # 'name': 'ImmutableDict' + # } + # } + # ] - def test_definition_of_definition(self, flask_workspace): - result = flask_workspace.definition("/flask/blueprints.py", 142, 8) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'flask' - }, - 'name': 'record_once', - 'container': 'flask.blueprints', - 'kind': 'def', - 'file': 'blueprints.py', - 'position': { - 'line': 142, - 'character': 8 - } - }, - 'location': { - 'uri': 'file:///flask/blueprints.py', - 'range': { - 'start': { - 'line': 142, - 'character': 8 - }, - 'end': { - 'line': 142, - 'character': 19 - } - } - } - } - ] + # def test_definition_of_definition(self, flask_workspace): + # result = flask_workspace.definition("/flask/blueprints.py", 142, 8) + # assert result == [ + # { + # 'symbol': { + # 'package': { + # 'name': 'flask' + # }, + # 'name': 'record_once', + # 'container': 'flask.blueprints', + # 'kind': 'def', + # 'file': 'blueprints.py', + # 'position': { + # 'line': 142, + # 'character': 8 + # } + # }, + # 'location': { + # 'uri': 'file:///flask/blueprints.py', + # 'range': { + # 'start': { + # 'line': 142, + # 'character': 8 + # }, + # 'end': { + # 'line': 142, + # 'character': 19 + # } + # } + # } + # } + # ] From e2142600ad387b0b55775b3270a93b295e9846c5 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 16:52:38 -0800 Subject: [PATCH 25/45] fix flask hover tests (the response from jedi seems to be correct, just a change in its behavior due to the update version) --- test/test_flask.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_flask.py b/test/test_flask.py index 935d350..804406b 100644 --- a/test/test_flask.py +++ b/test/test_flask.py @@ -245,7 +245,7 @@ def test_cross_repo_hover(self, flask_workspace): 'contents': [ { 'language': 'python', - 'value': 'class ImmutableDict(param type(self))' + 'value': 'class ImmutableDict(param args, param kwargs)' }, 'An immutable :class:`dict`.\n\n.. versionadded:: 0.5' ] @@ -345,7 +345,7 @@ def test_stdlib_hover(self, flask_workspace): 'contents': [ { 'language': 'python', - 'value': 'class timedelta(param type(self))'}, + 'value': 'class timedelta(param args, param kwargs)'}, 'Represent the difference between two datetime objects.\n\n' 'Supported operators:\n\n' '- add, subtract timedelta\n' From a807253d7c5113bb723bdb22fd8aa95958bdfe2a Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 17:19:39 -0800 Subject: [PATCH 26/45] add working local def tests for flask --- test/test_flask.py | 274 +++++++++++++++++++-------------------------- 1 file changed, 118 insertions(+), 156 deletions(-) diff --git a/test/test_flask.py b/test/test_flask.py index 804406b..fca8db8 100644 --- a/test/test_flask.py +++ b/test/test_flask.py @@ -80,164 +80,126 @@ def test_cross_package_hover(self, flask_workspace): 'value': 'def jsonify(param *args, param **kwargs)' } - # # TODO(aaron): the actual definition results have duplicates for some - # # reason ... maybe the TestFileSystem? - # def test_local_definition(self, flask_workspace): - # result = flask_workspace.definition("/flask/cli.py", 215, 15) - # symbol = { - # 'location': { - # 'range': { - # 'end': { - # 'character': 17, - # 'line': 34 - # }, - # 'start': { - # 'character': 4, - # 'line': 34 - # } - # }, - # 'uri': 'file:///flask/cli.py' - # }, - # 'symbol': { - # 'container': 'flask.cli', - # 'file': 'cli.py', - # 'kind': 'def', - # 'name': 'find_best_app', - # 'package': { - # 'name': 'flask' - # }, - # 'position': { - # 'character': 4, - # 'line': 34 - # } - # } - # } - # assert symbol in result + def test_local_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/cli.py", 215, 15) - # def test_cross_module_definition(self, flask_workspace): - # result = flask_workspace.definition("/flask/app.py", 220, 12) - # symbol = { - # 'location': { - # 'range': { - # 'end': { - # 'character': 43, - # 'line': 26 - # }, - # 'start': { - # 'character': 28, - # 'line': 26 - # } - # }, - # 'uri': 'file:///flask/app.py' - # }, - # 'symbol': { - # 'container': 'flask.app', - # 'file': 'app.py', - # 'kind': 'class', - # 'name': 'ConfigAttribute', - # 'package': { - # 'name': 'flask' - # }, - # 'position': { - # 'character': 28, - # 'line': 26 - # } - # } - # } - # assert symbol in result - - # def test_cross_package_definition(self, flask_workspace): - # result = flask_workspace.definition("/flask/__init__.py", 44, 15) - # symbol = { - # 'symbol': { - # 'package': { - # 'name': 'flask' - # }, - # 'name': 'jsonify', - # 'container': 'flask.json', - # 'kind': 'def', - # 'file': '__init__.py', - # 'position': { - # 'line': 202, - # 'character': 4 - # } - # }, - # 'location': { - # 'uri': 'file:///flask/json/__init__.py', - # 'range': { - # 'start': { - # 'line': 202, - # 'character': 4 - # }, - # 'end': { - # 'line': 202, - # 'character': 11 - # } - # } - # } - # } - # assert symbol in result + assert len(result) == 1 + definition = result[0] - # def test_local_package_import_definition(self, flask_workspace): - # result = flask_workspace.definition("/flask/__init__.py", 44, 10) - # assert result == [ - # { - # 'symbol': { - # 'package': { - # 'name': 'flask' - # }, - # 'name': 'json', - # 'container': 'flask.json', - # 'kind': 'module', - # 'file': '__init__.py', - # 'position': { - # 'line': 0, - # 'character': 0 - # } - # }, - # 'location': { - # 'uri': 'file:///flask/json/__init__.py', - # 'range': { - # 'start': { - # 'line': 0, - # 'character': 0 - # }, - # 'end': { - # 'line': 0, - # 'character': 4 - # } - # } - # } - # }, - # { - # 'symbol': { - # 'package': { - # 'name': 'flask' - # }, - # 'name': 'json', - # 'container': 'flask', - # 'kind': 'module', - # 'file': '__init__.py', - # 'position': { - # 'line': 40, - # 'character': 14 - # } - # }, - # 'location': { - # 'uri': 'file:///flask/__init__.py', - # 'range': { - # 'start': { - # 'line': 40, - # 'character': 14 - # }, - # 'end': { - # 'line': 40, - # 'character': 18 - # } - # } - # } - # }, - # ] + assert "location" in definition + assert definition["location"] == { + 'range': { + 'end': { + 'character': 17, + 'line': 34 + }, + 'start': { + 'character': 4, + 'line': 34 + } + }, + 'uri': 'file:///flask/cli.py' + } + + def test_cross_module_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/app.py", 220, 12) + + assert len(result) == 2 + assert all(["location" in d for d in result]) + + result_locations = [d["location"] for d in result] + + definition_location = { + 'uri': 'file:///flask/config.py', + 'range': { + 'start': { + 'line': 20, + 'character': 6 + }, + 'end': { + 'line': 20, + 'character': 21 + } + } + } + + # from the import statement at the top of the file + assignment_location = { + 'uri': 'file:///flask/app.py', + 'range': { + 'end': { + 'character': 43, + 'line': 26 + }, + 'start': { + 'character': 28, + 'line': 26 + } + }, + } + + assert definition_location in result_locations + assert assignment_location in result_locations + + def test_cross_package_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/__init__.py", 44, 15) + + assert len(result) == 1 + definition = result[0] + + assert definition["location"] == { + 'uri': 'file:///flask/json/__init__.py', + 'range': { + 'start': { + 'line': 202, + 'character': 4 + }, + 'end': { + 'line': 202, + 'character': 11 + } + } + } + + def test_local_package_import_definition(self, flask_workspace): + result = flask_workspace.definition("/flask/__init__.py", 44, 10) + + assert len(result) == 2 + assert all(["location" in d for d in result]) + + result_locations = [d["location"] for d in result] + + definition_location = { + 'uri': 'file:///flask/json/__init__.py', + 'range': { + 'start': { + 'line': 0, + 'character': 0 + }, + 'end': { + 'line': 0, + 'character': 4 + } + } + } + + # from the import statement directly above it + assignment_location = { + 'uri': 'file:///flask/__init__.py', + 'range': { + 'start': { + 'line': 40, + 'character': 14 + }, + 'end': { + 'line': 40, + 'character': 18 + } + } + } + + assert definition_location in result_locations + assert assignment_location in result_locations def test_cross_repo_hover(self, flask_workspace): result = flask_workspace.hover("/flask/app.py", 295, 20) From 16276b49fe8851fd2eca4574a6e6e00a04b22b9a Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 17:30:15 -0800 Subject: [PATCH 27/45] refactor install_ext_deps and add doc --- langserver/clone_workspace.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index fa1825a..852fd4d 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -41,12 +41,13 @@ def __init__(self, fs: FileSystem, project_root: str, log.debug("Setting Cloned Project path to %s", self.CLONED_PROJECT_PATH) + self.PROJECT_HAS_PIPFILE = False + # Clone the project from the provided filesystem into the local # cache - existing_pipfile = False for file_path in self.fs.walk(str(self.PROJECT_ROOT)): if file_path.endswith("Pipfile"): - existing_pipfile = True + self.PROJECT_HAS_PIPFILE = True cache_file_path = self.project_to_cache_path(file_path) @@ -54,15 +55,7 @@ def __init__(self, fs: FileSystem, project_root: str, file_contents = self.fs.open(file_path) cache_file_path.write_text(file_contents) - # Install 3rd party deps - - # pipenv creates the Pipfile automatically whenever it does anything - - # only install if the project had one to begin with - if existing_pipfile: - self._install_pipenv() - - self._install_pip() - self._install_setup_py() + self._install_external_dependencies() def get_module_info(self, raw_jedi_module_path): """ @@ -113,8 +106,23 @@ def project_to_cache_path(self, project_path): # strip the leading '/' so that we can join it properly return self.CLONED_PROJECT_PATH / project_path.lstrip("/") + def _install_external_dependencies(self): + """ + Installs the external dependencies for the project. + + Known limitations: + - doesn't handle installation files that aren't in the root of the workspace + - no error handling if any of the installation steps will prompt the user for whatever + reason + """ + self._install_setup_py() + self._install_pip() + self._install_pipenv() + def _install_pipenv(self): - if (self.CLONED_PROJECT_PATH / "Pipfile").exists(): + # pipenv creates the Pipfile automatically whenever it does anything - + # only install if the project had one to begin with + if (self.PROJECT_HAS_PIPFILE and (self.CLONED_PROJECT_PATH / "Pipfile").exists()): self.run_command("pipenv install -dev") def _install_pip(self): From a3e0c764090f742e2284b909b9e3b2e6b3fc543b Mon Sep 17 00:00:00 2001 From: ggilmore Date: Wed, 7 Mar 2018 17:31:10 -0800 Subject: [PATCH 28/45] add explicit str() call to install_pip --- langserver/clone_workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 852fd4d..3b67243 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -128,7 +128,7 @@ def _install_pipenv(self): def _install_pip(self): for requirements_file in self.CLONED_PROJECT_PATH.glob("*requirements.txt"): self.run_command( - "pip install -r {}".format(requirements_file.absolute())) + "pip install -r {}".format(str(requirements_file.absolute()))) def _install_setup_py(self): if (self.CLONED_PROJECT_PATH / "setup.py").exists(): From 14a27439d2b85d69537f178679a2a607a216fda5 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 09:20:24 -0800 Subject: [PATCH 29/45] assume file is text if the mimetype is none --- langserver/fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langserver/fs.py b/langserver/fs.py index 30fcecf..99735c6 100644 --- a/langserver/fs.py +++ b/langserver/fs.py @@ -213,7 +213,7 @@ def _walk(self, top: str): dirs.append(os.path.relpath(e, self.root)) else: file_type = mimetypes.guess_type(e)[0] - if file_type and file_type.startswith("text/"): + if file_type is None or file_type.startswith("text/"): files.append(os.path.relpath(e, self.root)) yield from files for d in dirs: From e0d520847426166c6f010a3e62dac65beb3beb22 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 09:20:40 -0800 Subject: [PATCH 30/45] fix / enhance _install_pipenv logic --- langserver/clone_workspace.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 3b67243..6f0e754 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -122,8 +122,11 @@ def _install_external_dependencies(self): def _install_pipenv(self): # pipenv creates the Pipfile automatically whenever it does anything - # only install if the project had one to begin with - if (self.PROJECT_HAS_PIPFILE and (self.CLONED_PROJECT_PATH / "Pipfile").exists()): - self.run_command("pipenv install -dev") + if self.PROJECT_HAS_PIPFILE: + if (self.CLONED_PROJECT_PATH / "Pipfile.lock").exists(): + self.run_command("pipenv install --dev --ignore-pipfile") + elif (self.CLONED_PROJECT_PATH / "Pipfile").exists(): + self.run_command("pipenv install --dev") def _install_pip(self): for requirements_file in self.CLONED_PROJECT_PATH.glob("*requirements.txt"): From 5518f8686ae14214d5b3ac44a72191b717b35d0f Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 09:20:55 -0800 Subject: [PATCH 31/45] add conflictingdeps test --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index f667b2c..5071c97 100644 --- a/Makefile +++ b/Makefile @@ -9,5 +9,6 @@ lint: test: # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_conflictingdeps.py -vv cd ./test && pipenv run pytest test_flask.py -vv cd ./test && pipenv run pytest test_fizzbuzz.py -vv From 0d10e5e60a6bb6af2f5f2f390065f11500ad7352 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 09:36:01 -0800 Subject: [PATCH 32/45] add back dep versioning tests The last test had to be changed since that repo has no installation file. This means that no ext dep is downloaded for that repo, so obviously there is no way for jedi to resolve the hover. --- Makefile | 1 + test/test_dep_versioning.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5071c97..38ae489 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ lint: test: # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_dep_versioning.py -vv cd ./test && pipenv run pytest test_conflictingdeps.py -vv cd ./test && pipenv run pytest test_flask.py -vv cd ./test && pipenv run pytest test_fizzbuzz.py -vv diff --git a/test/test_dep_versioning.py b/test/test_dep_versioning.py index 415f918..356771a 100644 --- a/test/test_dep_versioning.py +++ b/test/test_dep_versioning.py @@ -10,7 +10,6 @@ ("repos/dep_versioning_fixed", "this is version 0.1"), ("repos/dep_versioning_between", "this is version 0.4"), ("repos/dep_versioning_between_multiple", "this is version 0.4"), - ("repos/dep_versioning_none", "this is version 0.6") ]) def test_data(request): repo_path, expected_doc_string = request.param @@ -22,11 +21,22 @@ def test_data(request): workspace.exit() +@pytest.fixture +def workspace_no_installation_file(): + workspace = Harness("repos/dep_versioning_none") + workspace.initialize("repos/dep_versioning_none" + str(uuid.uuid4())) + yield workspace + workspace.exit() + + class TestDependencyVersioning: + def test_dep_download_specified_version(self, test_data): workspace, expected_doc_string = test_data + uri = "file:///test.py" character, line = 6, 2 + result = workspace.hover(uri, line, character) assert result == { 'contents': [ @@ -37,3 +47,20 @@ def test_dep_download_specified_version(self, test_data): expected_doc_string ] } + + def test_dep_no_installation_files(self, workspace_no_installation_file): + # no setup.py, *requirements.txt, or Pipfile present in the project + + uri = "file:///test.py" + character, line = 6, 2 + + result = workspace_no_installation_file.hover( + uri, line, character) + assert result == { + 'contents': [ + { + 'language': 'python', + 'value': 'testfunc' + } + ] + } From 4608a9c7e3d43d59a37685f933fbdc8f6c3f0d1d Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 09:38:27 -0800 Subject: [PATCH 33/45] add back modpkgsamename tests --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 38ae489..091d554 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ lint: test: # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_modpkgsamename.py -vv cd ./test && pipenv run pytest test_dep_versioning.py -vv cd ./test && pipenv run pytest test_conflictingdeps.py -vv cd ./test && pipenv run pytest test_flask.py -vv From cd058741b6d92e700f13754bcc131af95c00a8d5 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 09:50:10 -0800 Subject: [PATCH 34/45] ignore decoding errors when running workspace in tests --- langserver/clone_workspace.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 6f0e754..1e641a0 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -1,6 +1,6 @@ import logging from .config import GlobalConfig -from .fs import FileSystem +from .fs import FileSystem, TestFileSystem from shutil import rmtree import delegator from functools import lru_cache @@ -50,10 +50,17 @@ def __init__(self, fs: FileSystem, project_root: str, self.PROJECT_HAS_PIPFILE = True cache_file_path = self.project_to_cache_path(file_path) - - cache_file_path.parent.mkdir(parents=True, exist_ok=True) - file_contents = self.fs.open(file_path) - cache_file_path.write_text(file_contents) + try: + file_contents = self.fs.open(file_path) + cache_file_path.parent.mkdir(parents=True, exist_ok=True) + cache_file_path.write_text(file_contents) + except UnicodeDecodeError as e: + if isinstance(self.fs, TestFileSystem): + # assume that it's trying to write some non-text file, which + # should only happen when running tests + continue + else: + raise e self._install_external_dependencies() From ccb6396a488e30fed7691e30c4eeba00d93eada8 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 09:55:48 -0800 Subject: [PATCH 35/45] add back tensorflow tests --- Makefile | 1 + test/test_tensorflow_models.py | 69 +++++++++++++--------------------- 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index 091d554..b8c7590 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ lint: test: # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_tensorflow_models.py -vv cd ./test && pipenv run pytest test_modpkgsamename.py -vv cd ./test && pipenv run pytest test_dep_versioning.py -vv cd ./test && pipenv run pytest test_conflictingdeps.py -vv diff --git a/test/test_tensorflow_models.py b/test/test_tensorflow_models.py index 3b2f800..15ff15b 100644 --- a/test/test_tensorflow_models.py +++ b/test/test_tensorflow_models.py @@ -1,30 +1,27 @@ from .harness import Harness import uuid +import pytest -tensorflow_models_workspace = Harness("repos/tensorflow-models") -tensorflow_models_workspace.initialize( - "git://github.com/tensorflow/models?" + str(uuid.uuid4())) +@pytest.fixture +def workspace(): + tensorflow_models_workspace = Harness("repos/tensorflow-models") + tensorflow_models_workspace.initialize( + "git://github.com/tensorflow/models?" + str(uuid.uuid4())) + yield tensorflow_models_workspace + tensorflow_models_workspace.exit() -def test_namespace_package_definition(): - result = tensorflow_models_workspace.definition( - "/inception/inception/flowers_eval.py", 23, 22) - symbol = { - 'symbol': { - 'package': { - 'name': 'inception_eval' - }, - 'name': 'inception_eval', - 'container': 'inception_eval', - 'kind': 'module', - 'file': 'inception_eval.py', - 'position': { - 'line': 0, - 'character': 0 - } - }, - 'location': { +class TestTensorflowModels: + def test_namespace_package_definition(self, workspace): + result = workspace.definition( + "/inception/inception/flowers_eval.py", 23, 22) + + assert len(result) == 1 + definition = result[0] + + assert "location" in definition + definition["location"] == { 'uri': 'file:///inception/inception/inception_eval.py', 'range': { 'start': { @@ -37,28 +34,16 @@ def test_namespace_package_definition(): } } } - } - assert symbol in result + def test_ad_hoc_module_definition(self, workspace): + result = workspace.definition( + "/skip_thoughts/skip_thoughts/evaluate.py", 43, 26) -def test_ad_hoc_module_definition(): - result = tensorflow_models_workspace.definition( - "/skip_thoughts/skip_thoughts/evaluate.py", 43, 26) - symbol = { - 'symbol': { - 'package': { - 'name': 'skip_thoughts' - }, - 'name': 'encoder_manager', - 'container': 'skip_thoughts.encoder_manager', - 'kind': 'module', - 'file': 'encoder_manager.py', - 'position': { - 'line': 0, - 'character': 0 - } - }, - 'location': { + assert len(result) == 1 + definition = result[0] + + assert "location" in definition + definition["location"] == { 'uri': 'file:///skip_thoughts/skip_thoughts/encoder_manager.py', 'range': { 'start': { @@ -71,5 +56,3 @@ def test_ad_hoc_module_definition(): } } } - } - assert symbol in result From 7768ada7a55f3fa7276887f90f5e2fcd1d8528ac Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 10:20:21 -0800 Subject: [PATCH 36/45] add back graphql core test also delete the skipped test, which is literally a copy of the above passing test --- Makefile | 1 + test/test_graphql_core.py | 100 +++++++++++++++----------------------- 2 files changed, 39 insertions(+), 62 deletions(-) diff --git a/Makefile b/Makefile index b8c7590..6414bf0 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ lint: test: # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_graphql_core.py -vv cd ./test && pipenv run pytest test_tensorflow_models.py -vv cd ./test && pipenv run pytest test_modpkgsamename.py -vv cd ./test && pipenv run pytest test_dep_versioning.py -vv diff --git a/test/test_graphql_core.py b/test/test_graphql_core.py index ac819e7..3a1b843 100644 --- a/test/test_graphql_core.py +++ b/test/test_graphql_core.py @@ -3,76 +3,52 @@ import pytest -graphql_core_workspace = Harness("repos/graphql-core") -graphql_core_workspace.initialize( - "git://github.com/plangrid/graphql-core?" + str(uuid.uuid4())) +@pytest.fixture +def workspace(): + graphql_core_workspace = Harness("repos/graphql-core") + graphql_core_workspace.initialize( + "git://github.com/plangrid/graphql-core?" + str(uuid.uuid4())) + yield graphql_core_workspace + graphql_core_workspace.exit() -def test_relative_import_definition(): - result = graphql_core_workspace.definition("/graphql/__init__.py", 52, 8) - assert result == [ - { - 'symbol': { - 'package': { - 'name': 'graphql' - }, - 'name': 'GraphQLObjectType', - 'container': 'graphql.type.definition', - 'kind': 'class', - 'file': 'definition.py', - 'position': { +class TestGraphqlCore: + def test_relative_import_definition(self, workspace): + result = workspace.definition("/graphql/__init__.py", 52, 8) + + assert len(result) == 2 + assert all(["location" in d for d in result]) + + result_locations = [d["location"] for d in result] + + definition_location = { + 'uri': 'file:///graphql/type/definition.py', + 'range': { + 'start': { 'line': 138, 'character': 6 - } - }, - 'location': { - 'uri': 'file:///graphql/type/definition.py', - 'range': { - 'start': { - 'line': 138, - 'character': 6 - }, - 'end': { - 'line': 138, - 'character': 23 - } + }, + 'end': { + 'line': 138, + 'character': 23 } } - }, - { - 'symbol': { - 'package': { - 'name': 'graphql' - }, - 'name': 'GraphQLObjectType', - 'container': 'graphql.type', - 'kind': 'class', - 'file': '__init__.py', - 'position': { + } + + # re-exported in the __init__ file for the defining package + re_exported_location = { + 'uri': 'file:///graphql/type/__init__.py', + 'range': { + 'start': { 'line': 3, 'character': 4 - } - }, - 'location': { - 'uri': 'file:///graphql/type/__init__.py', - 'range': { - 'start': { - 'line': 3, - 'character': 4 - }, - 'end': { - 'line': 3, - 'character': 21 - } + }, + 'end': { + 'line': 3, + 'character': 21 } } - }, - ] - + } -# TODO(aaron): not all relative imports work; seems to be a Jedi bug, as -# it appears in other Jedi-based extensions too -@pytest.mark.skip(reason="Jedi bug") -def test_relative_import_definition_broken(): - result = graphql_core_workspace.definition("/graphql/__init__.py", 52, 8) - assert result == [] + assert definition_location in result_locations + assert re_exported_location in result_locations From 043720794620faa7fc2069906221a5c9a7e6da95 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 10:27:31 -0800 Subject: [PATCH 37/45] add back jedi tests --- Makefile | 1 + test/test_jedi.py | 105 ++++++++++++++++++++++------------------------ 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 6414bf0..21a1f40 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ lint: test: # pipenv run pytest test_langserver.py + cd ./test && pipenv run pytest test_jedi.py -vv cd ./test && pipenv run pytest test_graphql_core.py -vv cd ./test && pipenv run pytest test_tensorflow_models.py -vv cd ./test && pipenv run pytest test_modpkgsamename.py -vv diff --git a/test/test_jedi.py b/test/test_jedi.py index 4d39e66..c87bbd4 100644 --- a/test/test_jedi.py +++ b/test/test_jedi.py @@ -1,63 +1,60 @@ from .harness import Harness import uuid +import pytest -jedi_workspace = Harness("repos/jedi") -jedi_workspace.initialize( - "git://github.com/davidhalter/jedi?" + str(uuid.uuid4())) +@pytest.fixture +def workspace(): + jedi_workspace = Harness("repos/jedi") + jedi_workspace.initialize( + "git://github.com/davidhalter/jedi?" + str(uuid.uuid4())) + yield jedi_workspace + jedi_workspace.exit() -def test_x_packages(): - packages = jedi_workspace.x_packages() - assert packages - result = None - for p in packages: - if "package" in p and p["package"] == {"name": "jedi"}: - result = p - break - assert result - assert "package" in result and result["package"] == {'name': 'jedi'} - assert "dependencies" in result - dep_names = {d["attributes"]["name"] for d in result["dependencies"]} - assert dep_names == { - 'pytest', - 'not_in_sys_path', - 'numpy', - '__main__', - 'import_tree', - 'recurse_class2', - 'not_existing', - 'psutil', - 'rename1', - 'local_module', - 'objgraph', - 'django', - 'cpython', - 'pylab', - 'docopt', - 'recurse_class1', - 'psycopg2', - 'not_existing_nested', - 'pygments' - } +class TestJedi: + # def test_x_packages(self, workspace): + # packages = workspace.x_packages() + # assert packages + # result = None + # for p in packages: + # if "package" in p and p["package"] == {"name": "jedi"}: + # result = p + # break + # assert result + # assert "package" in result and result["package"] == {'name': 'jedi'} + # assert "dependencies" in result + # dep_names = {d["attributes"]["name"] for d in result["dependencies"]} + # assert dep_names == { + # 'pytest', + # 'not_in_sys_path', + # 'numpy', + # '__main__', + # 'import_tree', + # 'recurse_class2', + # 'not_existing', + # 'psutil', + # 'rename1', + # 'local_module', + # 'objgraph', + # 'django', + # 'cpython', + # 'pylab', + # 'docopt', + # 'recurse_class1', + # 'psycopg2', + # 'not_existing_nested', + # 'pygments' + # } -def test_absolute_import_definiton(): - result = jedi_workspace.definition("/jedi/api/__init__.py", 28, 26) - symbol = { - 'symbol': { - 'package': { - 'name': 'jedi' - }, - 'name': 'Evaluator', - 'container': 'jedi.evaluate', - 'kind': 'class', - 'file': '__init__.py', - 'position': { - 'line': 86, - 'character': 6 - } - }, - 'location': { + def test_absolute_import_definition(self, workspace): + result = workspace.definition("/jedi/api/__init__.py", 28, 26) + + assert len(result) == 1 + definition = result[0] + + assert "location" in definition + assert definition["location"] == { 'uri': 'file:///jedi/evaluate/__init__.py', 'range': { 'start': { @@ -70,5 +67,3 @@ def test_absolute_import_definiton(): } } } - } - assert symbol in result From 854f40fcdd4c184f5fd2fd834af7641198bbbea5 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 10:33:11 -0800 Subject: [PATCH 38/45] comment out remaning tests that don't deal with local defs or hovers --- Makefile | 9 +- test/test_clone_workspace.py | 34 ++--- test/test_global_variables.py | 48 +++---- test/test_thefuck.py | 232 +++++++++++++++++----------------- 4 files changed, 158 insertions(+), 165 deletions(-) diff --git a/Makefile b/Makefile index 21a1f40..dcf2458 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,4 @@ lint: test: # pipenv run pytest test_langserver.py - cd ./test && pipenv run pytest test_jedi.py -vv - cd ./test && pipenv run pytest test_graphql_core.py -vv - cd ./test && pipenv run pytest test_tensorflow_models.py -vv - cd ./test && pipenv run pytest test_modpkgsamename.py -vv - cd ./test && pipenv run pytest test_dep_versioning.py -vv - cd ./test && pipenv run pytest test_conflictingdeps.py -vv - cd ./test && pipenv run pytest test_flask.py -vv - cd ./test && pipenv run pytest test_fizzbuzz.py -vv + cd ./test && pipenv run pytest test_*.py -vv diff --git a/test/test_clone_workspace.py b/test/test_clone_workspace.py index 278b779..430cad9 100644 --- a/test/test_clone_workspace.py +++ b/test/test_clone_workspace.py @@ -1,22 +1,22 @@ -from langserver.clone_workspace import CloneWorkspace -from langserver.fs import TestFileSystem -import delegator -import pytest +# from langserver.clone_workspace import CloneWorkspace +# from langserver.fs import TestFileSystem +# import delegator +# import pytest -@pytest.fixture() -def test_data(): - repoPath = "repos/fizzbuzz_service" - workspace = CloneWorkspace(TestFileSystem(repoPath), repoPath, repoPath) - yield (workspace, repoPath) +# @pytest.fixture() +# def test_data(): +# repoPath = "repos/fizzbuzz_service" +# workspace = CloneWorkspace(TestFileSystem(repoPath), repoPath, repoPath) +# yield (workspace, repoPath) -class TestCloningWorkspace: - def test_clone(self, test_data): - workspace, repoPath = test_data +# class TestCloningWorkspace: +# def test_clone(self, test_data): +# workspace, repoPath = test_data - c = delegator.run("diff -r {} {}".format(repoPath, - workspace.CLONED_PROJECT_PATH)) - assert c.out == "" - assert c.err == "" - assert c.return_code == 0 +# c = delegator.run("diff -r {} {}".format(repoPath, +# workspace.CLONED_PROJECT_PATH)) +# assert c.out == "" +# assert c.err == "" +# assert c.return_code == 0 diff --git a/test/test_global_variables.py b/test/test_global_variables.py index 7b9727c..760b0c2 100644 --- a/test/test_global_variables.py +++ b/test/test_global_variables.py @@ -1,28 +1,28 @@ -from .harness import Harness +# from .harness import Harness -workspace = Harness("repos/global-variables") -workspace.initialize("") +# workspace = Harness("repos/global-variables") +# workspace.initialize("") -def test_name_definition(): - # The __name__ global variable should resolve to a symbol without - # a corresponding location - uri = "file:///name_global.py" - line, col = 0, 4 +# def test_name_definition(): +# # The __name__ global variable should resolve to a symbol without +# # a corresponding location +# uri = "file:///name_global.py" +# line, col = 0, 4 - result = workspace.definition(uri, line, col) - assert result == [ - { - 'symbol': - { - 'package': { - 'name': 'name_global' - }, - 'name': '__name__', - 'container': 'name_global', - 'kind': 'instance', - 'file': 'name_global.py' - }, - 'location': None - } - ] +# result = workspace.definition(uri, line, col) +# assert result == [ +# { +# 'symbol': +# { +# 'package': { +# 'name': 'name_global' +# }, +# 'name': '__name__', +# 'container': 'name_global', +# 'kind': 'instance', +# 'file': 'name_global.py' +# }, +# 'location': None +# } +# ] diff --git a/test/test_thefuck.py b/test/test_thefuck.py index fb1fb62..f0bc4e3 100644 --- a/test/test_thefuck.py +++ b/test/test_thefuck.py @@ -1,120 +1,120 @@ -from .harness import Harness -import uuid +# from .harness import Harness +# import uuid -thefuck_workspace = Harness("repos/thefuck") -thefuck_workspace.initialize( - "git://github.com/nvbn/thefuck?" + str(uuid.uuid4())) +# thefuck_workspace = Harness("repos/thefuck") +# thefuck_workspace.initialize( +# "git://github.com/nvbn/thefuck?" + str(uuid.uuid4())) -# make sure that the resulting locations don't refer to files on the local -# filesystem -def test_local_references(): - result = thefuck_workspace.references( - "/thefuck/argument_parser.py", 22, 21) - assert result == [ - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 18, - 'character': 21 - }, - 'end': { - 'line': 18, - 'character': 33 - } - } - }, - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 22, - 'character': 21 - }, - 'end': { - 'line': 22, - 'character': 33 - } - } - }, - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 27, - 'character': 21 - }, - 'end': { - 'line': 27, - 'character': 33 - } - } - }, - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 32, - 'character': 21 - }, - 'end': { - 'line': 32, - 'character': 33 - } - } - }, - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 36, - 'character': 21 - }, - 'end': { - 'line': 36, - 'character': 33 - } - } - }, - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 40, - 'character': 21 - }, - 'end': { - 'line': 40, - 'character': 33 - } - } - }, - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 48, - 'character': 14 - }, - 'end': { - 'line': 48, - 'character': 26 - } - } - }, - { - 'uri': 'file:///thefuck/argument_parser.py', - 'range': { - 'start': { - 'line': 52, - 'character': 14 - }, - 'end': { - 'line': 52, - 'character': 26 - } - } - } - ] +# # make sure that the resulting locations don't refer to files on the local +# # filesystem +# def test_local_references(): +# result = thefuck_workspace.references( +# "/thefuck/argument_parser.py", 22, 21) +# assert result == [ +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 18, +# 'character': 21 +# }, +# 'end': { +# 'line': 18, +# 'character': 33 +# } +# } +# }, +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 22, +# 'character': 21 +# }, +# 'end': { +# 'line': 22, +# 'character': 33 +# } +# } +# }, +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 27, +# 'character': 21 +# }, +# 'end': { +# 'line': 27, +# 'character': 33 +# } +# } +# }, +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 32, +# 'character': 21 +# }, +# 'end': { +# 'line': 32, +# 'character': 33 +# } +# } +# }, +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 36, +# 'character': 21 +# }, +# 'end': { +# 'line': 36, +# 'character': 33 +# } +# } +# }, +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 40, +# 'character': 21 +# }, +# 'end': { +# 'line': 40, +# 'character': 33 +# } +# } +# }, +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 48, +# 'character': 14 +# }, +# 'end': { +# 'line': 48, +# 'character': 26 +# } +# } +# }, +# { +# 'uri': 'file:///thefuck/argument_parser.py', +# 'range': { +# 'start': { +# 'line': 52, +# 'character': 14 +# }, +# 'end': { +# 'line': 52, +# 'character': 26 +# } +# } +# } +# ] From e5807b2ecccfc420d9db662695ae0c116e472f49 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 10:33:55 -0800 Subject: [PATCH 39/45] remove vv flag --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dcf2458..065570d 100644 --- a/Makefile +++ b/Makefile @@ -9,4 +9,4 @@ lint: test: # pipenv run pytest test_langserver.py - cd ./test && pipenv run pytest test_*.py -vv + cd ./test && pipenv run pytest test_*.py From d3467503ca1f56a7cde998a4f2a7e90e3065bd67 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 11:46:32 -0800 Subject: [PATCH 40/45] HACK: force pipenv to create venvs (right now for tests) --- langserver/clone_workspace.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 1e641a0..6b45972 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -6,6 +6,7 @@ from functools import lru_cache from enum import Enum from pathlib import Path +import os log = logging.getLogger(__name__) @@ -183,8 +184,17 @@ def run_command(self, command, no_prefix=False, **kwargs): ''' kwargs["cwd"] = self.CLONED_PROJECT_PATH - # add pipenv prefix if not no_prefix: + + # HACK: this is to get our spawned pipenv to keep creating + # venvs even if the language server itself is running inside one + # See: + # https://github.com/pypa/pipenv/blob/4e8deda9cbf2a658ab40ca31cc6e249c0b53d6f4/pipenv/environments.py#L58 + + env = kwargs.get("env", os.environ.copy()) + env["VIRTUAL_ENV"] = "" + kwargs["env"] = env + if type(command) is str: command = "pipenv run {}".format(command) else: From cf7fe9787587fe16d13aa6b1b012c41910b7719f Mon Sep 17 00:00:00 2001 From: ggilmore Date: Thu, 8 Mar 2018 13:03:45 -0800 Subject: [PATCH 41/45] more strict checking for Pipfile at repo root --- langserver/clone_workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py index 6b45972..44ab6cd 100644 --- a/langserver/clone_workspace.py +++ b/langserver/clone_workspace.py @@ -47,7 +47,7 @@ def __init__(self, fs: FileSystem, project_root: str, # Clone the project from the provided filesystem into the local # cache for file_path in self.fs.walk(str(self.PROJECT_ROOT)): - if file_path.endswith("Pipfile"): + if file_path == "/Pipfile": self.PROJECT_HAS_PIPFILE = True cache_file_path = self.project_to_cache_path(file_path) From ebbd7dc5636bce6b3311d76141717f6d38e56052 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Fri, 9 Mar 2018 09:22:17 -0800 Subject: [PATCH 42/45] rm old workspace, rename cloneworkspace to workspace --- langserver/clone_workspace.py | 210 ------------ langserver/langserver.py | 11 +- langserver/workspace.py | 583 ++++++++++------------------------ 3 files changed, 180 insertions(+), 624 deletions(-) delete mode 100644 langserver/clone_workspace.py diff --git a/langserver/clone_workspace.py b/langserver/clone_workspace.py deleted file mode 100644 index 44ab6cd..0000000 --- a/langserver/clone_workspace.py +++ /dev/null @@ -1,210 +0,0 @@ -import logging -from .config import GlobalConfig -from .fs import FileSystem, TestFileSystem -from shutil import rmtree -import delegator -from functools import lru_cache -from enum import Enum -from pathlib import Path -import os - -log = logging.getLogger(__name__) - - -class CloneWorkspace: - - def __init__(self, fs: FileSystem, project_root: str, - original_root_path: str= ""): - - self.fs = fs - self.PROJECT_ROOT = Path(project_root) - self.repo = None - self.hash = None - - if original_root_path.startswith( - "git://") and "?" in original_root_path: - repo_and_hash = original_root_path.split("?") - self.repo = repo_and_hash[0] - original_root_path = self.repo - self.hash = repo_and_hash[1] - - if original_root_path.startswith("git://"): - original_root_path = original_root_path[6:] - - # turn the original root path into something that can be used as a - # file/path name or cache key - self.key = original_root_path.replace("/", ".").replace("\\", ".") - if self.hash: - self.key = ".".join((self.key, self.hash)) - - self.CLONED_PROJECT_PATH = GlobalConfig.CLONED_PROJECT_PATH / self.key - - log.debug("Setting Cloned Project path to %s", - self.CLONED_PROJECT_PATH) - - self.PROJECT_HAS_PIPFILE = False - - # Clone the project from the provided filesystem into the local - # cache - for file_path in self.fs.walk(str(self.PROJECT_ROOT)): - if file_path == "/Pipfile": - self.PROJECT_HAS_PIPFILE = True - - cache_file_path = self.project_to_cache_path(file_path) - try: - file_contents = self.fs.open(file_path) - cache_file_path.parent.mkdir(parents=True, exist_ok=True) - cache_file_path.write_text(file_contents) - except UnicodeDecodeError as e: - if isinstance(self.fs, TestFileSystem): - # assume that it's trying to write some non-text file, which - # should only happen when running tests - continue - else: - raise e - - self._install_external_dependencies() - - def get_module_info(self, raw_jedi_module_path): - """ - Given an absolute module path provided from jedi, - returns a tuple of (module_kind, rel_path) where: - - module_kind: The category that the module belongs to - (module is declared inside the project, module is a std_lib module, etc.) - - rel_path: The path of the module relative to the context - which it's defined in. e.x: if module_kind == 'PROJECT', - rel_path is the path of the module relative to the project's root. - """ - - module_path = Path(raw_jedi_module_path) - - if self.CLONED_PROJECT_PATH in module_path.parents: - return (ModuleKind.PROJECT, module_path.relative_to(self.CLONED_PROJECT_PATH)) - - if GlobalConfig.PYTHON_PATH in module_path.parents: - return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(GlobalConfig.PYTHON_PATH)) - - venv_path = self.VENV_PATH / "lib" - if venv_path in module_path.parents: - # The python libraries in a venv are stored under - # VENV_LOCATION/lib/(some_python_version) - - python_version = module_path.relative_to(venv_path).parts[0] - - venv_lib_path = venv_path / python_version - ext_dep_path = venv_lib_path / "site-packages" - - if ext_dep_path in module_path.parents: - return (ModuleKind.EXTERNAL_DEPENDENCY, module_path.relative_to(ext_dep_path)) - - return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(venv_lib_path)) - - return (ModuleKind.UNKNOWN, module_path) - - def project_to_cache_path(self, project_path): - """ - Translates a path from the root of the project to the equivalent path in - the local cache. - - e.x.: '/a/b.py' -> '/python-cloned-projects-cache/project_name/a/b.py' - """ - - # strip the leading '/' so that we can join it properly - return self.CLONED_PROJECT_PATH / project_path.lstrip("/") - - def _install_external_dependencies(self): - """ - Installs the external dependencies for the project. - - Known limitations: - - doesn't handle installation files that aren't in the root of the workspace - - no error handling if any of the installation steps will prompt the user for whatever - reason - """ - self._install_setup_py() - self._install_pip() - self._install_pipenv() - - def _install_pipenv(self): - # pipenv creates the Pipfile automatically whenever it does anything - - # only install if the project had one to begin with - if self.PROJECT_HAS_PIPFILE: - if (self.CLONED_PROJECT_PATH / "Pipfile.lock").exists(): - self.run_command("pipenv install --dev --ignore-pipfile") - elif (self.CLONED_PROJECT_PATH / "Pipfile").exists(): - self.run_command("pipenv install --dev") - - def _install_pip(self): - for requirements_file in self.CLONED_PROJECT_PATH.glob("*requirements.txt"): - self.run_command( - "pip install -r {}".format(str(requirements_file.absolute()))) - - def _install_setup_py(self): - if (self.CLONED_PROJECT_PATH / "setup.py").exists(): - self.run_command("python setup.py install") - - @property - @lru_cache() - def VENV_PATH(self): - """ - The absolute path of the virtual environment created for this workspace. - """ - self.ensure_venv_created() - venv_path = self.run_command("pipenv --venv").out.rstrip() - return Path(venv_path) - - def cleanup(self): - log.info("Removing project's virtual environment %s", self.VENV_PATH) - self.remove_venv() - - log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) - rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) - - def ensure_venv_created(self): - ''' - This runs a noop pipenv command, which will - create the venv if it doesn't exist as a side effect. - ''' - self.run_command("true") - - def remove_venv(self): - self.run_command("pipenv --rm", no_prefix=True) - - def run_command(self, command, no_prefix=False, **kwargs): - ''' - Runs the given command inside the context - of the project: - - Context means: - - the command's cwd is the cached project directory - - the projects virtual environment is loaded into - the command's environment - ''' - kwargs["cwd"] = self.CLONED_PROJECT_PATH - - if not no_prefix: - - # HACK: this is to get our spawned pipenv to keep creating - # venvs even if the language server itself is running inside one - # See: - # https://github.com/pypa/pipenv/blob/4e8deda9cbf2a658ab40ca31cc6e249c0b53d6f4/pipenv/environments.py#L58 - - env = kwargs.get("env", os.environ.copy()) - env["VIRTUAL_ENV"] = "" - kwargs["env"] = env - - if type(command) is str: - command = "pipenv run {}".format(command) - else: - command = ["pipenv", "run"].append(command) - - return delegator.run(command, **kwargs) - - -class ModuleKind(Enum): - PROJECT = 1 - STANDARD_LIBRARY = 2 - EXTERNAL_DEPENDENCY = 3 - UNKNOWN = 4 diff --git a/langserver/langserver.py b/langserver/langserver.py index 9120ba0..2bad0c3 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -11,11 +11,10 @@ from .fs import LocalFileSystem, RemoteFileSystem from .jedi import RemoteJedi from .jsonrpc import JSONRPC2Connection, ReadWriter, TCPReadWriter -# from .workspace import Workspace +from .workspace import Workspace, ModuleKind from .symbols import extract_symbols, workspace_symbols from .definitions import targeted_symbol from .references import get_references -from .clone_workspace import CloneWorkspace, ModuleKind log = logging.getLogger(__name__) @@ -178,9 +177,7 @@ def serve_initialize(self, request): # Sourcegraph also passes in a rootUri which has commit information originalRootUri = params.get("originalRootUri") or params.get( "originalRootPath") or "" - # self.workspace = Workspace( - # self.fs, self.root_path, originalRootUri, pip_args) - self.workspace = CloneWorkspace( + self.workspace = Workspace( self.fs, self.root_path, originalRootUri) return { @@ -202,8 +199,8 @@ def test_initialize(self, request, fs): self.fs = fs self.streaming = False - self.workspace = CloneWorkspace(self.fs, self.root_path, - params["originalRootPath"]) + self.workspace = Workspace(self.fs, self.root_path, + params["originalRootPath"]) return { "capabilities": { diff --git a/langserver/workspace.py b/langserver/workspace.py index 77e7d81..00e5cdf 100644 --- a/langserver/workspace.py +++ b/langserver/workspace.py @@ -1,65 +1,23 @@ -from .config import GlobalConfig -from .fs import FileSystem, LocalFileSystem, FileException -from .imports import get_imports -from .fetch import fetch_dependency -from .requirements_parser import parse_requirements, get_version_specifier_for_pkg -from typing import Dict, Set, List - import logging -import sys +from .config import GlobalConfig +from .fs import FileSystem, TestFileSystem +from shutil import rmtree +import delegator +from functools import lru_cache +from enum import Enum +from pathlib import Path import os -import os.path -import shutil -import threading -import opentracing -import jedi._compatibility - log = logging.getLogger(__name__) -class DummyFile: - def __init__(self, contents): - self.contents = contents - - def read(self): - return self.contents - - def close(self): - pass - - -class Module: - def __init__(self, - name: str, - qualified_name: str, - path: str, - is_package: bool=False, - is_external: bool=False, - is_stdlib: bool=False, - is_native: bool=False, - is_namespace_package: bool=False): - self.name = name - self.qualified_name = qualified_name - self.path = path - self.is_package = is_package - self.is_external = is_external - self.is_stdlib = is_stdlib - self.is_native = is_native - self.is_namespace_package = is_namespace_package - - def __repr__(self): - return "PythonModule({}, {})".format(self.name, self.path) - - class Workspace: def __init__(self, fs: FileSystem, project_root: str, - original_root_path: str= "", pip_args: List[str]=[]): + original_root_path: str= ""): - self.pip_args = pip_args - self.project_packages = set() - self.PROJECT_ROOT = project_root + self.fs = fs + self.PROJECT_ROOT = Path(project_root) self.repo = None self.hash = None @@ -70,8 +28,6 @@ def __init__(self, fs: FileSystem, project_root: str, original_root_path = self.repo self.hash = repo_and_hash[1] - self.is_stdlib = (self.repo == GlobalConfig.STDLIB_REPO_URL) - if original_root_path.startswith("git://"): original_root_path = original_root_path[6:] @@ -81,361 +37,174 @@ def __init__(self, fs: FileSystem, project_root: str, if self.hash: self.key = ".".join((self.key, self.hash)) - # TODO: allow different Python versions per project/workspace - self.PYTHON_PATH = GlobalConfig.PYTHON_PATH - self.PACKAGES_PATH = os.path.join( - GlobalConfig.PACKAGES_PARENT, self.key) - log.debug("Setting Python path to %s", self.PYTHON_PATH) - log.debug("Setting package path to %s", self.PACKAGES_PATH) + self.CLONED_PROJECT_PATH = GlobalConfig.CLONED_PROJECT_PATH / self.key - self.fs = fs - self.local_fs = LocalFileSystem() - self.source_paths = {path for path in self.fs.walk( - self.PROJECT_ROOT) if path.endswith(".py")} - self.project = {} - self.stdlib = {} - self.dependencies = {} - self.module_paths = {} - # keep track of which package folders have been indexed, since we fetch - # and index new folders on-demand - self.indexed_folders = set() - self.indexing_lock = threading.Lock() - # keep track of which packages we've tried to fetch, so we don't keep - # trying if they were unfetchable - self.fetched = set() - - self.index_project() - - for n in sys.builtin_module_names: - # TODO: figure out how to provide code intelligence for compiled-in - # modules - self.stdlib[n] = "native" - if "nt" not in self.stdlib: - # this is missing on non-Windows systems; add it so we don't try to - # fetch it - self.stdlib["nt"] = "native" - - if os.path.exists(self.PYTHON_PATH): - log.debug("Indexing standard library at %s", self.PYTHON_PATH) - self.index_dependencies( - self.stdlib, self.PYTHON_PATH, is_stdlib=True) - else: - log.warning("Standard library not found at %s", self.PYTHON_PATH) - - # if the dependencies are already cached from a previous session, - # go ahead and index them, otherwise just create the folder and let - # them be fetched on-demand - if os.path.exists(self.PACKAGES_PATH): - self.index_external_modules() - else: - os.makedirs(self.PACKAGES_PATH) + log.debug("Setting Cloned Project path to %s", + self.CLONED_PROJECT_PATH) - def cleanup(self): - log.info("Removing package cache %s", self.PACKAGES_PATH) - shutil.rmtree(self.PACKAGES_PATH, True) - - def index_dependencies(self, - index: Dict[str, Module], - library_path: str, - is_stdlib: bool=False, - breadcrumb: str=None): - """Given a root library path (e.g., the Python root path or the dist- - packages root path), this method traverses it recursively and indexes - all the packages and modules contained therein. It constructs a mapping - from the fully qualified module name to a Module object containing the - metadata that Jedi needs. - - :param index: the dictionary that should be used to store this index (will be modified) - :param library_path: the root path containing the modules and packages to be indexed - :param is_stdlib: flag indicating whether this invocation is indexing the standard library - :param breadcrumb: should be omitted by the caller; this method uses it to keep track of - the fully qualified module name + self.PROJECT_HAS_PIPFILE = False + + # Clone the project from the provided filesystem into the local + # cache + for file_path in self.fs.walk(str(self.PROJECT_ROOT)): + if file_path == "/Pipfile": + self.PROJECT_HAS_PIPFILE = True + + cache_file_path = self.project_to_cache_path(file_path) + try: + file_contents = self.fs.open(file_path) + cache_file_path.parent.mkdir(parents=True, exist_ok=True) + cache_file_path.write_text(file_contents) + except UnicodeDecodeError as e: + if isinstance(self.fs, TestFileSystem): + # assume that it's trying to write some non-text file, which + # should only happen when running tests + continue + else: + raise e + + self._install_external_dependencies() + + def get_module_info(self, raw_jedi_module_path): """ - parent, this = os.path.split(library_path) - basename, extension = os.path.splitext(this) - if Workspace.is_package( - library_path) or extension == ".py" and this != "__init__.py": - qualified_name = ".".join( - (breadcrumb, basename)) if breadcrumb else basename - elif extension == ".so": - basename = basename.split(".")[0] - qualified_name = ".".join( - (breadcrumb, basename)) if breadcrumb else basename - else: - qualified_name = breadcrumb - - if os.path.isdir(library_path): - # don't index third-party packages installed in our python path - if library_path == os.path.join(self.PYTHON_PATH, "site-packages"): - return - - # recursively index this folder - for child in os.listdir(library_path): - self.index_dependencies(index, os.path.join( - library_path, child), is_stdlib, qualified_name) - elif this == "__init__.py": - # we're already inside a package - module_name = os.path.basename(parent) - the_module = Module(module_name, - qualified_name, - library_path, - True, - True, - is_stdlib) - index[qualified_name] = the_module - self.module_paths[os.path.abspath(the_module.path)] = the_module - - elif extension == ".py": - # just a regular non-package module - the_module = Module(basename, - qualified_name, - library_path, - False, - True, - is_stdlib) - index[qualified_name] = the_module - self.module_paths[os.path.abspath(the_module.path)] = the_module - - elif extension == ".so": - # native module -- mark it as such and report a warning or - # something - the_module = Module(basename, - qualified_name, - "", - False, - True, - is_stdlib, - True) - index[qualified_name] = the_module - self.module_paths[os.path.abspath(the_module.path)] = the_module - - def index_project(self): - """This method traverses all the project files (starting with - self.PROJECT_ROOT) and indexes all the packages and modules contained - therein. - - It constructs a mapping from the fully qualified module name to - a Module object containing the metadata that Jedi needs. Because - it only has a flat list of paths/uris to work with (as opposed - to being able to walk the file tree top-down), it does some - extra work to figure out the qualified names of each module. + Given an absolute module path provided from jedi, + returns a tuple of (module_kind, rel_path) where: + + module_kind: The category that the module belongs to + (module is declared inside the project, module is a std_lib module, etc.) + + rel_path: The path of the module relative to the context + which it's defined in. e.x: if module_kind == 'PROJECT', + rel_path is the path of the module relative to the project's root. """ - all_paths = list(self.fs.walk(self.PROJECT_ROOT)) - - # TODO: maybe try to exec setup.py with a sandboxed global env and builtins dict or - # something pre-compute the set of all packages in this project -- this will be useful - # when trying to figure out each module's qualified name, as well as the packages that - # are exported by this project - package_paths = {} - top_level_modules = set() - for path in all_paths: - folder, filename = os.path.split(path) - if filename == "__init__.py": - package_paths[folder] = True - if folder == "/"\ - and filename.endswith(".py")\ - and filename not in {"__init__.py", "setup.py", "tests.py", "test.py"}: - basename, extension = os.path.splitext(filename) - top_level_modules.add(basename) - - # figure out this project's exports (for xpackages) - for path in package_paths: - if path.startswith("/"): - path = path[1:] - else: - continue - path_components = path.split("/") - if len(path_components) > 1: - continue - else: - self.project_packages.add(path_components[0]) - - # if this is the case, then the exports must be in top-level Python files - if not self.project_packages: - for m in top_level_modules: - self.project_packages.add(m) - - # now index all modules and packages, taking care to compute their qualified names - # correctly (can be tricky depending on how the folders are nested, and whether they have - # '__init__.py's or not - for path in all_paths: - folder, filename = os.path.split(path) - basename, ext = os.path.splitext(filename) - if filename == "__init__.py": - parent, this = os.path.split(folder) - elif filename.endswith(".py"): - parent = folder - this = basename - else: - continue - qualified_name_components = [this] - # A module's qualified name should only contain the names of its enclosing folders that - # are packages (i.e., that contain an '__init__.py'), not the names of *all* its - # enclosing folders. Hence, the following loop only accumulates qualified name - # components until it encounters a folder that isn't in the pre-computed - # set of packages. - while parent and parent != "/" and parent in package_paths: - parent, this = os.path.split(parent) - qualified_name_components.append(this) - qualified_name_components.reverse() - qualified_name = ".".join(qualified_name_components) - if filename == "__init__.py": - module_name = os.path.basename(folder) - the_module = Module(module_name, qualified_name, path, True) - self.project[qualified_name] = the_module - self.module_paths[the_module.path] = the_module - elif ext == ".py" and not basename.startswith("__") and not basename.endswith("__"): - the_module = Module(basename, qualified_name, path) - self.project[qualified_name] = the_module - self.module_paths[the_module.path] = the_module - - def find_stdlib_module(self, qualified_name: str) -> Module: - return self.stdlib.get(qualified_name, None) - - def find_project_module(self, qualified_name: str) -> Module: - return self.project.get(qualified_name, None) - - def find_external_module(self, qualified_name: str) -> Module: - package_name = qualified_name.split(".")[0] - if package_name not in self.fetched: - self.indexing_lock.acquire() - self.fetched.add(package_name) - specifier = self.get_ext_pkg_version_specifier(package_name) - fetch_dependency(package_name, specifier, self.PACKAGES_PATH, self.pip_args) - self.index_external_modules() - self.indexing_lock.release() - the_module = self.dependencies.get(qualified_name, None) - if the_module and the_module.is_native: - raise NotImplementedError("Unable to analyze native modules") - else: - return the_module - - def get_ext_pkg_version_specifier(self, package_name): - """Gets the version specifier to use after parsing the project's - requirements file. - - (See limitations and caveats in .requirements_parser.parse_requirements() - and .requirements_parser.get_version_specifier_for_pkg()). - - If a requirements file isn't found at the root of the repo, or if there was an error - parsing it, a string representing that any version is allowed is returned. + + module_path = Path(raw_jedi_module_path) + + if self.CLONED_PROJECT_PATH in module_path.parents: + return (ModuleKind.PROJECT, module_path.relative_to(self.CLONED_PROJECT_PATH)) + + if GlobalConfig.PYTHON_PATH in module_path.parents: + return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(GlobalConfig.PYTHON_PATH)) + + venv_path = self.VENV_PATH / "lib" + if venv_path in module_path.parents: + # The python libraries in a venv are stored under + # VENV_LOCATION/lib/(some_python_version) + + python_version = module_path.relative_to(venv_path).parts[0] + + venv_lib_path = venv_path / python_version + ext_dep_path = venv_lib_path / "site-packages" + + if ext_dep_path in module_path.parents: + return (ModuleKind.EXTERNAL_DEPENDENCY, module_path.relative_to(ext_dep_path)) + + return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(venv_lib_path)) + + return (ModuleKind.UNKNOWN, module_path) + + def project_to_cache_path(self, project_path): + """ + Translates a path from the root of the project to the equivalent path in + the local cache. + + e.x.: '/a/b.py' -> '/python-cloned-projects-cache/project_name/a/b.py' + """ + + # strip the leading '/' so that we can join it properly + return self.CLONED_PROJECT_PATH / project_path.lstrip("/") + + def _install_external_dependencies(self): + """ + Installs the external dependencies for the project. + + Known limitations: + - doesn't handle installation files that aren't in the root of the workspace + - no error handling if any of the installation steps will prompt the user for whatever + reason + """ + self._install_setup_py() + self._install_pip() + self._install_pipenv() + + def _install_pipenv(self): + # pipenv creates the Pipfile automatically whenever it does anything - + # only install if the project had one to begin with + if self.PROJECT_HAS_PIPFILE: + if (self.CLONED_PROJECT_PATH / "Pipfile.lock").exists(): + self.run_command("pipenv install --dev --ignore-pipfile") + elif (self.CLONED_PROJECT_PATH / "Pipfile").exists(): + self.run_command("pipenv install --dev") + + def _install_pip(self): + for requirements_file in self.CLONED_PROJECT_PATH.glob("*requirements.txt"): + self.run_command( + "pip install -r {}".format(str(requirements_file.absolute()))) + + def _install_setup_py(self): + if (self.CLONED_PROJECT_PATH / "setup.py").exists(): + self.run_command("python setup.py install") + + @property + @lru_cache() + def VENV_PATH(self): + """ + The absolute path of the virtual environment created for this workspace. """ - pkg_specifiers_map = {} - try: - pkg_specifiers_map = parse_requirements( - "/requirements.txt", self.fs) - except (FileException, FileNotFoundError) as e: - log.warning( - "error parsing requirements file for {}, err: {}".format( - self.PROJECT_ROOT, e)) - - return get_version_specifier_for_pkg(package_name, pkg_specifiers_map) - - def index_external_modules(self): - for path in os.listdir(self.PACKAGES_PATH): - if path not in self.indexed_folders: - self.index_dependencies( - self.dependencies, os.path.join(self.PACKAGES_PATH, path)) - self.indexed_folders.add(path) - - def open_module_file(self, the_module: Module, - parent_span: opentracing.Span): - if the_module.path not in self.source_paths: - return None - elif the_module.is_external: - return DummyFile(self.local_fs.open(the_module.path, parent_span)) - else: - return DummyFile(self.fs.open(the_module.path, parent_span)) - - def get_module_by_path(self, path: str) -> Module: - return self.module_paths.get(path, None) - - def get_modules(self, qualified_name: str) -> List[Module]: - project_module = self.project.get(qualified_name, None) - external_module = self.find_external_module(qualified_name) - stdlib_module = self.stdlib.get(qualified_name, None) - return list( - filter(None, [project_module, external_module, stdlib_module])) - - def get_dependencies(self, parent_span: opentracing.Span) -> list: - top_level_stdlib = {p.split(".")[0] for p in self.stdlib} - top_level_imports = get_imports( - self.fs, self.PROJECT_ROOT, parent_span) - stdlib_imports = top_level_imports & top_level_stdlib - external_imports = top_level_imports - top_level_stdlib - self.project_packages - dependencies = [{"attributes": {"name": n}} for n in external_imports] - if stdlib_imports: - dependencies.append({ - "attributes": { - "name": "cpython", - "repoURL": "git://github.com/python/cpython" - } - }) - return dependencies - - def get_package_information(self, parent_span: opentracing.Span) -> list: - if self.is_stdlib: - return [{ - "package": {"name": "cpython"}, - "dependencies": [] - }] - else: - return [ - { - "package": {"name": p}, - # multiple packages in the project share the same - # dependencies - "dependencies": self.get_dependencies(parent_span) - } for p in self.project_packages - ] - - # finds a project module using the newer, more dynamic import rules detailed in PEP 420 - # (see https://www.python.org/dev/peps/pep-0420/) - def find_internal_module( - self, name: str, qualified_name: str, dirs: List[str]): - module_paths = [] - for parent in dirs: - if os.path.join(parent, name, "__init__.py") in self.source_paths: - # there's a folder at this level that implements a package with - # the name we're looking for - module_path = os.path.join(parent, name, "__init__.py") - module_file = DummyFile(self.fs.open(module_path)) - return module_file, module_path, True - elif (os.path.basename(parent) == name and - os.path.join(parent, "__init__.py") in self.source_paths): - # we're already in a package with the name we're looking for - module_path = os.path.join(parent, "__init__.py") - module_file = DummyFile(self.fs.open(module_path)) - return module_file, module_path, True - elif os.path.join(parent, name + ".py") in self.source_paths: - # there's a file at this level that implements a module with - # the name we're looking for - module_path = os.path.join(parent, name + ".py") - module_file = DummyFile(self.fs.open(module_path)) - return module_file, module_path, False - elif self.folder_exists(os.path.join(parent, name)): - # there's a folder at this level that implements a namespace - # package with the name we're looking for - module_paths.append(os.path.join(parent, name)) - elif self.folder_exists(parent) and os.path.basename(parent) == name: - # we're already in a namespace package with the name we're - # looking for - module_paths.append(parent) - if not module_paths: - return None, None, None - return None, jedi._compatibility.ImplicitNSInfo( - qualified_name, module_paths), False - - def folder_exists(self, name): - for path in self.source_paths: - if os.path.commonpath((name, path)) == name: - return True - return False - - @staticmethod - def is_package(path: str) -> bool: - return os.path.isdir(path) and "__init__.py" in os.listdir(path) - - @staticmethod - def get_top_level_package_names(index: Dict[str, Module]) -> Set[str]: - return {name.split(".")[0] for name in index} + self.ensure_venv_created() + venv_path = self.run_command("pipenv --venv").out.rstrip() + return Path(venv_path) + + def cleanup(self): + log.info("Removing project's virtual environment %s", self.VENV_PATH) + self.remove_venv() + + log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) + rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) + + def ensure_venv_created(self): + ''' + This runs a noop pipenv command, which will + create the venv if it doesn't exist as a side effect. + ''' + self.run_command("true") + + def remove_venv(self): + self.run_command("pipenv --rm", no_prefix=True) + + def run_command(self, command, no_prefix=False, **kwargs): + ''' + Runs the given command inside the context + of the project: + + Context means: + - the command's cwd is the cached project directory + - the projects virtual environment is loaded into + the command's environment + ''' + kwargs["cwd"] = self.CLONED_PROJECT_PATH + + if not no_prefix: + + # HACK: this is to get our spawned pipenv to keep creating + # venvs even if the language server itself is running inside one + # See: + # https://github.com/pypa/pipenv/blob/4e8deda9cbf2a658ab40ca31cc6e249c0b53d6f4/pipenv/environments.py#L58 + + env = kwargs.get("env", os.environ.copy()) + env["VIRTUAL_ENV"] = "" + kwargs["env"] = env + + if type(command) is str: + command = "pipenv run {}".format(command) + else: + command = ["pipenv", "run"].append(command) + + return delegator.run(command, **kwargs) + + +class ModuleKind(Enum): + PROJECT = 1 + STANDARD_LIBRARY = 2 + EXTERNAL_DEPENDENCY = 3 + UNKNOWN = 4 From b17dae21d4fca2e2b99a64bc566925daa4523ea2 Mon Sep 17 00:00:00 2001 From: ggilmore Date: Fri, 9 Mar 2018 12:29:55 -0800 Subject: [PATCH 43/45] wip - support multiple projects in workspace --- langserver/jedi.py | 5 +- langserver/langserver.py | 5 +- langserver/workspace.py | 123 ++++++++++++++++++++++++++++++--------- 3 files changed, 101 insertions(+), 32 deletions(-) diff --git a/langserver/jedi.py b/langserver/jedi.py index df58c8a..a23cfee 100644 --- a/langserver/jedi.py +++ b/langserver/jedi.py @@ -33,10 +33,11 @@ def _new_script_impl(self, parent_span, *args, **kwargs): if self.workspace is not None: path = self.workspace.project_to_cache_path(path) + project = self.workspace.find_project_for_path(path) environment = None - for env in jedi.find_virtualenvs([self.workspace.VENV_PATH], safe=False): - if env._base_path == self.workspace.VENV_PATH: + for env in jedi.find_virtualenvs([project.VENV_PATH], safe=False): + if env._base_path == project.VENV_PATH: environment = env break diff --git a/langserver/langserver.py b/langserver/langserver.py index 2bad0c3..0e4a878 100644 --- a/langserver/langserver.py +++ b/langserver/langserver.py @@ -354,12 +354,15 @@ def serve_x_definition(self, request): if not defs: return results + project = self.workspace.find_project_for_path( + self.workspace.project_to_cache_path(path)) + for d in defs: # TODO: handle case where a def doesn't have a module_path if not d.module_path: continue - module_kind, rel_module_path = self.workspace.get_module_info( + module_kind, rel_module_path = project.get_module_info( d.module_path) if module_kind != ModuleKind.PROJECT: diff --git a/langserver/workspace.py b/langserver/workspace.py index 00e5cdf..a12f97e 100644 --- a/langserver/workspace.py +++ b/langserver/workspace.py @@ -42,13 +42,9 @@ def __init__(self, fs: FileSystem, project_root: str, log.debug("Setting Cloned Project path to %s", self.CLONED_PROJECT_PATH) - self.PROJECT_HAS_PIPFILE = False - # Clone the project from the provided filesystem into the local # cache for file_path in self.fs.walk(str(self.PROJECT_ROOT)): - if file_path == "/Pipfile": - self.PROJECT_HAS_PIPFILE = True cache_file_path = self.project_to_cache_path(file_path) try: @@ -63,8 +59,83 @@ def __init__(self, fs: FileSystem, project_root: str, else: raise e + self.project = Project(self.CLONED_PROJECT_PATH, + self.CLONED_PROJECT_PATH) + + def find_project_for_path(self, path): + return self.project.find_project_for_path(path) + + def project_to_cache_path(self, project_path): + """ + Translates a path from the root of the project to the equivalent path in + the local cache. + + e.x.: '/a/b.py' -> '/python-cloned-projects-cache/project_name/a/b.py' + """ + + # strip the leading '/' so that we can join it properly + return self.CLONED_PROJECT_PATH / project_path.lstrip("/") + + def cleanup(self): + self.project.cleanup() + + +class Project: + def __init__(self, workspace_root_dir, project_root_dir): + self.WORKSPACE_ROOT_DIR = workspace_root_dir + self.PROJECT_ROOT_DIR = project_root_dir + self.sub_projects = self._find_subprojects(project_root_dir) self._install_external_dependencies() + def _find_subprojects(self, current_dir): + ''' + Returns a map containing the top-level subprojects contained inside + this project, keyed by the absolute path to the subproject. + ''' + sub_projects = {} + + top_level_folders = ( + child for child in current_dir.iterdir() if child.is_dir()) + + for folder in top_level_folders: + + def gen_len(generator): + return sum(1 for _ in generator) + + # do any subfolders contain installation files? + if any(gen_len(folder.glob(pattern)) for pattern in INSTALLATION_FILE_PATTERNS): + sub_projects[folder] = Project(self.WORKSPACE_ROOT_DIR, folder) + else: + sub_projects.update(self._find_subprojects(folder)) + + return sub_projects + + def find_project_for_path(self, path): + ''' + Returns the deepest project instance that contains this path. + + ''' + if path.is_file(): + folder = path.parent + else: + folder = path + + if folder == self.PROJECT_ROOT_DIR: + return self + + # If the project_dir isn't an ancestor of the folder, + # there is no way it or any of its subprojects + # could contain this path + if self.PROJECT_ROOT_DIR not in folder.parents: + return None + + for sub_project in self.sub_projects.values(): + deepest_project = sub_project.find_project_for_path(path) + if deepest_project is not None: + return deepest_project + + return self + def get_module_info(self, raw_jedi_module_path): """ Given an absolute module path provided from jedi, @@ -80,8 +151,8 @@ def get_module_info(self, raw_jedi_module_path): module_path = Path(raw_jedi_module_path) - if self.CLONED_PROJECT_PATH in module_path.parents: - return (ModuleKind.PROJECT, module_path.relative_to(self.CLONED_PROJECT_PATH)) + if self.PROJECT_ROOT_DIR in module_path.parents: + return (ModuleKind.PROJECT, module_path.relative_to(self.WORKSPACE_ROOT_DIR)) if GlobalConfig.PYTHON_PATH in module_path.parents: return (ModuleKind.STANDARD_LIBRARY, module_path.relative_to(GlobalConfig.PYTHON_PATH)) @@ -103,17 +174,6 @@ def get_module_info(self, raw_jedi_module_path): return (ModuleKind.UNKNOWN, module_path) - def project_to_cache_path(self, project_path): - """ - Translates a path from the root of the project to the equivalent path in - the local cache. - - e.x.: '/a/b.py' -> '/python-cloned-projects-cache/project_name/a/b.py' - """ - - # strip the leading '/' so that we can join it properly - return self.CLONED_PROJECT_PATH / project_path.lstrip("/") - def _install_external_dependencies(self): """ Installs the external dependencies for the project. @@ -128,21 +188,18 @@ def _install_external_dependencies(self): self._install_pipenv() def _install_pipenv(self): - # pipenv creates the Pipfile automatically whenever it does anything - - # only install if the project had one to begin with - if self.PROJECT_HAS_PIPFILE: - if (self.CLONED_PROJECT_PATH / "Pipfile.lock").exists(): - self.run_command("pipenv install --dev --ignore-pipfile") - elif (self.CLONED_PROJECT_PATH / "Pipfile").exists(): - self.run_command("pipenv install --dev") + if (self.PROJECT_ROOT_DIR / "Pipfile.lock").exists(): + self.run_command("pipenv install --dev --ignore-pipfile") + elif (self.PROJECT_ROOT_DIR / "Pipfile").exists(): + self.run_command("pipenv install --dev") def _install_pip(self): - for requirements_file in self.CLONED_PROJECT_PATH.glob("*requirements.txt"): + for requirements_file in self.PROJECT_ROOT_DIR.glob(REQUIREMENTS_GLOB_PATTERN): self.run_command( "pip install -r {}".format(str(requirements_file.absolute()))) def _install_setup_py(self): - if (self.CLONED_PROJECT_PATH / "setup.py").exists(): + if (self.PROJECT_ROOT_DIR / "setup.py").exists(): self.run_command("python setup.py install") @property @@ -156,11 +213,14 @@ def VENV_PATH(self): return Path(venv_path) def cleanup(self): + for sub_project in self.sub_projects.values(): + sub_project.cleanup() + log.info("Removing project's virtual environment %s", self.VENV_PATH) self.remove_venv() - log.info("Removing cloned project cache %s", self.CLONED_PROJECT_PATH) - rmtree(self.CLONED_PROJECT_PATH, ignore_errors=True) + log.info("Removing cloned project cache %s", self.PROJECT_ROOT_DIR) + rmtree(self.PROJECT_ROOT_DIR, ignore_errors=True) def ensure_venv_created(self): ''' @@ -182,7 +242,7 @@ def run_command(self, command, no_prefix=False, **kwargs): - the projects virtual environment is loaded into the command's environment ''' - kwargs["cwd"] = self.CLONED_PROJECT_PATH + kwargs["cwd"] = self.PROJECT_ROOT_DIR if not no_prefix: @@ -203,6 +263,11 @@ def run_command(self, command, no_prefix=False, **kwargs): return delegator.run(command, **kwargs) +REQUIREMENTS_GLOB_PATTERN = "*requirements.txt" + +INSTALLATION_FILE_PATTERNS = ["Pipfile", REQUIREMENTS_GLOB_PATTERN, "setup.py"] + + class ModuleKind(Enum): PROJECT = 1 STANDARD_LIBRARY = 2 From 72bf615435b53af04090b1011af30a47f8775f5b Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Fri, 16 Mar 2018 14:59:23 +0200 Subject: [PATCH 44/45] rm .cache --- test/.cache/v/cache/lastfailed | 40 ---------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 test/.cache/v/cache/lastfailed diff --git a/test/.cache/v/cache/lastfailed b/test/.cache/v/cache/lastfailed deleted file mode 100644 index b35d3ee..0000000 --- a/test/.cache/v/cache/lastfailed +++ /dev/null @@ -1,40 +0,0 @@ -{ - "test_clone_workspace.py::TestCloningWorkspace::()::test_clone": true, - "test_conflictingdeps.py::TestConflictingDependencies::()::test_hover_packages_conflicting_module_names": true, - "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data0]": true, - "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data1]": true, - "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data2]": true, - "test_dep_versioning.py::TestDependencyVersioning::()::test_dep_download_specified_version[test_data3]": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_local_references": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_std_lib_definition": true, - "test_fizzbuzz.py::TestFizzBuzzWorkspace::()::test_x_references": true, - "test_fizzbuzz.py::test_cross_package_definition": true, - "test_fizzbuzz.py::test_cross_package_import_definition": true, - "test_fizzbuzz.py::test_local_defintion": true, - "test_fizzbuzz.py::test_local_package_cross_module_definition": true, - "test_fizzbuzz.py::test_local_package_cross_module_import_definition": true, - "test_fizzbuzz.py::test_local_references": true, - "test_fizzbuzz.py::test_std_lib_definition": true, - "test_fizzbuzz.py::test_x_packages": true, - "test_flask.py::test_cross_module_definition": true, - "test_flask.py::test_cross_module_hover": true, - "test_flask.py::test_cross_package_definition": true, - "test_flask.py::test_cross_package_hover": true, - "test_flask.py::test_cross_repo_definition": true, - "test_flask.py::test_cross_repo_hover": true, - "test_flask.py::test_cross_repo_import_definition": true, - "test_flask.py::test_definition_of_definition": true, - "test_flask.py::test_local_definition": true, - "test_flask.py::test_local_package_import_definition": true, - "test_flask.py::test_local_references": true, - "test_flask.py::test_stdlib_definition": true, - "test_flask.py::test_x_packages": true, - "test_global_variables.py::test_name_definition": true, - "test_graphql_core.py::test_relative_import_definition": true, - "test_jedi.py::test_absolute_import_definiton": true, - "test_jedi.py::test_x_packages": true, - "test_modpkgsamename.py::TestExtPkgHasModuleWithSameName::()::test_hover_pkg_module_same_name": true, - "test_tensorflow_models.py::test_ad_hoc_module_definition": true, - "test_tensorflow_models.py::test_namespace_package_definition": true, - "test_thefuck.py::test_local_references": true -} \ No newline at end of file From dd19b44eec8e7b33fda7b83f2a8abce6dade4045 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Fri, 16 Mar 2018 16:00:34 +0200 Subject: [PATCH 45/45] use latest pytest --- Pipfile | 4 +- Pipfile.lock | 158 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 111 insertions(+), 51 deletions(-) diff --git a/Pipfile b/Pipfile index b1de99a..42f0b78 100644 --- a/Pipfile +++ b/Pipfile @@ -17,8 +17,8 @@ jedi = {git = "git://github.com/davidhalter/jedi.git", editable = true, ref = "3 [dev-packages] -py = "==1.4.34" -pytest = "==3.2.3" +py = "*" +pytest = "*" "autopep8" = "*" "flake8" = "*" autoflake = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 0046817..cd7ffcc 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "49c740bc7aadcb217862ec6482768eb43546b003d4ade69ce68cef8c8b248954" + "sha256": "f53e802702fe9e4eab140d9a1a3ebb134ec736f0762d6810cd4a5d7375e4eb02" }, "pipfile-spec": 6, "requires": { @@ -22,10 +22,17 @@ ], "version": "==2.2.1" }, + "certifi": { + "hashes": [ + "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", + "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" + ], + "version": "==2018.1.18" + }, "delegator.py": { "hashes": [ - "sha256:58f3ea6fe36680e1d828e2e66e52844b826f186409dfee4436e42351b0e699fe", - "sha256:2d46966a7f484d271b09e2646eae1e9acadc4fdf2cb760c142f073e81c927d8d" + "sha256:2d46966a7f484d271b09e2646eae1e9acadc4fdf2cb760c142f073e81c927d8d", + "sha256:58f3ea6fe36680e1d828e2e66e52844b826f186409dfee4436e42351b0e699fe" ], "version": "==0.1.0" }, @@ -42,10 +49,10 @@ }, "lightstep": { "hashes": [ - "sha256:ef41c37e307b33e1afb8349052dde25801bfebb6950fb9133112b7dce159c0c3", - "sha256:c4919005f190ce1b4528ffec0cc43fa1d0f52fe0833ab9d5cd95e51db5413e27" + "sha256:31fc800bbce7f69cf2e0af4bf8a0c539c805200b03ffa08cfd91db47a9414bff", + "sha256:51534ba6516c4c4e61ae763793a87fedf11840ae85a67ddf85b574b5d115da30" ], - "version": "==3.0.10" + "version": "==3.0.11" }, "opentracing": { "hashes": [ @@ -55,48 +62,53 @@ }, "parso": { "hashes": [ - "sha256:a7bb86fe0844304869d1c08e8bd0e52be931228483025c422917411ab82d628a", - "sha256:5815f3fe254e5665f3c5d6f54f086c2502035cb631a91341591b5a564203cffb" + "sha256:5815f3fe254e5665f3c5d6f54f086c2502035cb631a91341591b5a564203cffb", + "sha256:a7bb86fe0844304869d1c08e8bd0e52be931228483025c422917411ab82d628a" ], "version": "==0.1.1" }, - "pew": { - "hashes": [ - "sha256:a5256586e169199cb2213225261b6bdd7b657c309dcf4a02b61994308b313d4d", - "sha256:b8312728526c9010295c88215c95a1b1731fdbd1a568f728e069932bd0545611" - ], - "version": "==1.1.2" - }, "pexpect": { "hashes": [ - "sha256:6ff881b07aff0cb8ec02055670443f784434395f90c3285d2ae470f921ade52a", - "sha256:67b85a1565968e3d5b5e7c9283caddc90c3947a2625bed1905be27bd5a03e47d" + "sha256:67b85a1565968e3d5b5e7c9283caddc90c3947a2625bed1905be27bd5a03e47d", + "sha256:6ff881b07aff0cb8ec02055670443f784434395f90c3285d2ae470f921ade52a" ], "version": "==4.4.0" }, "pipenv": { "hashes": [ - "sha256:0796cb5078b00a26b9332a87ed137efa0c21e585dbf945d5943022e9b35e1663" + "sha256:94b3604874bbd40b2c2a3b769214c3a92681feeff3493c7d416e2d3ce38068bd" ], - "version": "==11.0.2" + "version": "==11.8.0" }, "protobuf": { "hashes": [ - "sha256:11788df3e176f44e0375fe6361342d7258a457b346504ea259a21b77ffc18a90", - "sha256:50c24f0d00b7efb3a72ae638ddc118e713cfe8cef40527afe24f7ebcb878e46d", - "sha256:41661f9a442eba2f1967f15333ebe9ecc7e7c51bcbaa2972303ad33a4ca0168e", - "sha256:06ec363b74bceb7d018f2171e0892f03ab6816530e2b0f77d725a58264551e48", - "sha256:b20f861b55efd8206428c13e017cc8e2c34b40b2a714446eb202bbf0ff7597a6", - "sha256:c1f9c36004a7ae6f1ce4a23f06070f6b07f57495f251851aa15cc4da16d08378", - "sha256:4d2e665410b0a278d2eb2c0a529ca2366bb325eb2ae34e189a826b71fb1b28cd", - "sha256:95b78959572de7d7fafa3acb718ed71f482932ddddddbd29ba8319c10639d863" + "sha256:09879a295fd7234e523b62066223b128c5a8a88f682e3aff62fb115e4a0d8be0", + "sha256:14813a3421ff0144e8d4e81ed83a3fbe350d8d85cbe480bf2e81cf45e8083e0d", + "sha256:18a4a387e8378dbbd53ebe9cc925ea2fe2a7b98c497833ea345803cb53b885d9", + "sha256:24c1cc840b4832a909bbeac664fd8f878cf72b8ab97bfe4fb82a156c3f1f0e15", + "sha256:40c943a8ffb3501164da1d2b537ad2e33d08daf81fbb3e9073bf291726a24467", + "sha256:41e916354265d2f54b95e454305c98f90bb30fafb817119540753e67f193de57", + "sha256:59610aeb5ade675106dca26c771814a1aa63bf2b3780584853e3dd447ed5c52f", + "sha256:59ff8a204aa2ef98d6c25c2adffb13dda81bb4ac6ffb0829c92e801241b6477b", + "sha256:64a3600d2a531d7c516c371efa431035ce501ab8425dcc8bdb99eddf5a4d34c9", + "sha256:6e1c0972462ce9dc4d2860d533487b39f89de00b3f30b99c31a6b3e8fbf8b787", + "sha256:75e1a7b12248a98b620ffbda3e41767aa2ae57c7cc553a12407a48c44f58f2e7", + "sha256:84ed523853c82c76dd1dfd15f31de2d66fa7cb22a48aa42dbc32465868d7e4af", + "sha256:87908d494be2b46a55de5e55ca11d9a2508b59b035c1b0549c3b692a77f57a7b", + "sha256:88c7958dad426920a43af58c5805d2de860a33f82d47f5a102af25f2788682c7", + "sha256:8ba58356fc40ed7749c73eeae3d86f6a9e756ba1ae5f5833990b237b7d61ba09", + "sha256:94d159e2bbbe4df1b5f0715965e284f2156ce127a7d521a3dcbdd38e945bc4c0", + "sha256:c4d531e745168c16fc7abff12922c491d34f4063c1b49fe5417b72be869f5df6", + "sha256:e457146bb9f997736460b10b2f2a9284603db4bbd60c8c431b5b4b309efbe036", + "sha256:e774cd03628c0b2f850a09a8c005fe6113f97e37f6df07a7b20221dc1ee4efd3", + "sha256:ec51286554eceebcf169a3a8604861e113d28fc98094dcbedc6067f058478917" ], - "version": "==3.5.1" + "version": "==3.5.2" }, "ptyprocess": { "hashes": [ - "sha256:e8c43b5eee76b2083a9badde89fd1bbce6c8942d1045146e100b7b5e014f4f1a", - "sha256:e64193f0047ad603b71f202332ab5527c5e52aa7c8b609704fc28c0dc20c4365" + "sha256:e64193f0047ad603b71f202332ab5527c5e52aa7c8b609704fc28c0dc20c4365", + "sha256:e8c43b5eee76b2083a9badde89fd1bbce6c8942d1045146e100b7b5e014f4f1a" ], "version": "==0.5.2" }, @@ -105,15 +117,10 @@ "git": "git://github.com/sourcegraph/requirements-parser.git", "ref": "69f1a9cb916b2995843c3ea9b988da46c9dd65c7" }, - "requirements-parser": { - "editable": true, - "git": "git://github.com/sourcegraph/requirements-parser.git", - "ref": "69f1a9cb916b2995843c3ea9b988da46c9dd65c7" - }, "six": { "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], "version": "==1.11.0" }, @@ -125,8 +132,8 @@ }, "virtualenv": { "hashes": [ - "sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1fc9fbb7b481e5c85b0", - "sha256:02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a" + "sha256:02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a", + "sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1fc9fbb7b481e5c85b0" ], "version": "==15.1.0" }, @@ -139,6 +146,13 @@ } }, "develop": { + "attrs": { + "hashes": [ + "sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9", + "sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450" + ], + "version": "==17.4.0" + }, "autoflake": { "hashes": [ "sha256:a74d684a7a02654f74582addc24a3016c06809316cc140457a4fe93a1e6ed131" @@ -151,19 +165,52 @@ ], "version": "==1.3.4" }, + "colorama": { + "hashes": [ + "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", + "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" + ], + "markers": "sys_platform == 'win32'", + "version": "==0.3.9" + }, + "configparser": { + "hashes": [ + "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" + ], + "markers": "python_version < '3.2'", + "version": "==3.5.0" + }, "docformatter": { "hashes": [ "sha256:036eba7c12669dc67c0ccaa3c40e2add3cc729af6a5fa4a2a54517bc09e10237" ], "version": "==1.0" }, + "enum34": { + "hashes": [ + "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", + "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", + "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", + "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + ], + "markers": "python_version < '3.4'", + "version": "==1.1.6" + }, "flake8": { "hashes": [ - "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37", - "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0" + "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0", + "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37" ], "version": "==3.5.0" }, + "funcsigs": { + "hashes": [ + "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", + "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" + ], + "markers": "python_version < '3.0'", + "version": "==1.0.2" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -171,17 +218,23 @@ ], "version": "==0.6.1" }, + "pluggy": { + "hashes": [ + "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff" + ], + "version": "==0.6.0" + }, "py": { "hashes": [ - "sha256:2ccb79b01769d99115aa600d7eed99f524bf752bba8f041dc1c184853514655a", - "sha256:0f2d585d22050e90c7d293b6451c83db097df77871974d90efd5a30dc12fcde3" + "sha256:8cca5c229d225f8c1e3085be4fcf306090b00850fefad892f9d96c7b6e2f310f", + "sha256:ca18943e28235417756316bfada6cd96b23ce60dd532642690dcfdaba988a76d" ], - "version": "==1.4.34" + "version": "==1.5.2" }, "pycodestyle": { "hashes": [ - "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9", - "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766" + "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", + "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9" ], "version": "==2.3.1" }, @@ -194,10 +247,17 @@ }, "pytest": { "hashes": [ - "sha256:81a25f36a97da3313e1125fce9e7bbbba565bc7fec3c5beb14c262ddab238ac1", - "sha256:27fa6617efc2869d3e969a3e75ec060375bfb28831ade8b5cdd68da3a741dc3c" + "sha256:062027955bccbc04d2fcd5d79690947e018ba31abe4c90b2c6721abec734261b", + "sha256:117bad36c1a787e1a8a659df35de53ba05f9f3398fb9e4ac17e80ad5903eb8c5" + ], + "version": "==3.4.2" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" ], - "version": "==3.2.3" + "version": "==1.11.0" }, "untokenize": { "hashes": [