Read requirements.txt at the root of the repository (#45)

Author: ggilmore

Pipfile

@@ -8,6 +8,7 @@ name = "pypi"
 [packages]
 
 jedi = {git = "git://github.com/sourcegraph/jedi.git", editable = true, ref = "9a3e7256df2e6099207fd7289141885ec17ebec7"}
+requirements = {git = "git://github.com/sourcegraph/requirements-parser.git", editable = true, ref = "69f1a9cb916b2995843c3ea9b988da46c9dd65c7"}
 opentracing = "*"
 lightstep = "*"
 

Pipfile.lock

@@ -10,11 +10,12 @@
 log = logging.getLogger(__name__)
 
 
-def fetch_dependency(module_name: str, install_path: str):
+def fetch_dependency(module_name: str, specifier: str, install_path: str):
     """
     Shells out to PIP in order to download and unzip the named package into the specified path. This method only runs
     `pip download`, NOT `pip install`, so it's presumably safe.
     :param module_name: the name of the package to download
+    :param specifier: the version specifier for the package
     :param install_path: the path in which to install the downloaded package
     """
     with tempfile.TemporaryDirectory() as download_folder:
@@ -23,9 +24,9 @@ def fetch_dependency(module_name: str, install_path: str):
 
         index_url = os.environ.get('INDEX_URL')
         if index_url is not None:
-            result = pip.main(["download", "--no-deps", "-i", index_url, "-d", download_folder, module_name])
+            result = pip.main(["download", "--no-deps", "-i", index_url, "-d", download_folder, module_name+specifier])
         else:
-            result = pip.main(["download", "--no-deps", "-d", download_folder, module_name])
+            result = pip.main(["download", "--no-deps", "-d", download_folder, module_name+specifier])
         if result != pip.status_codes.SUCCESS:
             log.error("Unable to fetch package %s", module_name)
             return
@@ -45,3 +46,4 @@ def fetch_dependency(module_name: str, install_path: str):
                 result = subprocess.run(["tar", "-C", install_path, "-xjf", thing_abs])
             else:
                 log.warning("Unrecognized package file: %s", thing, exc_info=True)
+

langserver/fetch.py

@@ -0,0 +1,40 @@
+from requirements import parse
+
+def parse_requirements(req_path, file_system):
+    """
+    Parses the pip requirements file located at req_path. Returns a map of package names 
+    to their version specifiers.
+
+    :param req_path: the path to the pip requirements file. Throws a FileNotFound or a FileException if
+    req_path is not valid
+
+    :param file_system: the file system to use to open the requirements file @ req_path and any other
+    recursive calls. 
+
+    Known limitations:
+
+    - All requirements files with that use the '--find-links', '--index-url', '--extra-index-url'
+    or '--no-index' flags are ignored. 
+
+    - All requirements that don't use a version specifier (e.x. django>=1.5 ) are ignored. 
+    """
+    req_string = file_system.open(req_path)
+    requirements = parse(req_string, current_path = req_path, file_system = file_system)
+    return { req.name:req.specs for req in requirements if req.specifier}
+
+def get_version_specifier_for_pkg(pkg, pkg_specifiers_map):
+    """
+    Returns the specifier string to use for a given requirement. If 
+    pkg has no corresponding entry in the pkg_specifiers_map, a string 
+    representing that any version is allowed is returned. 
+
+    :param pkg: the name of the package to get the version specifier for
+    :param pkg_specifiers_map: a map of packages to their respective version specifiers 
+    from a parsed requirements file 
+    """
+
+    specifier_strs = []
+    for spec in pkg_specifiers_map.get(pkg, [""]):
+        specifier_strs.append("".join(spec))
+    
+    return ",".join(specifier_strs)

langserver/requirements_parser.py

@@ -1,7 +1,8 @@
 from .config import GlobalConfig
-from .fs import FileSystem, LocalFileSystem
+from .fs import FileSystem, LocalFileSystem, FileException
 from .imports import get_imports
 from .fetch import fetch_dependency
+from .requirements_parser import parse_requirements, get_version_specifier_for_pkg
 from typing import Dict, Set, List
 
 import logging
@@ -272,7 +273,8 @@ def find_external_module(self, qualified_name: str) -> Module:
         if package_name not in self.fetched:
             self.indexing_lock.acquire()
             self.fetched.add(package_name)
-            fetch_dependency(package_name, self.PACKAGES_PATH)
+            specifier = self.get_ext_pkg_version_specifier(package_name)
+            fetch_dependency(package_name, specifier, self.PACKAGES_PATH)
             self.index_external_modules()
             self.indexing_lock.release()
         the_module = self.dependencies.get(qualified_name, None)
@@ -281,6 +283,25 @@ def find_external_module(self, qualified_name: str) -> Module:
         else:
             return the_module
 
+    def get_ext_pkg_version_specifier(self, package_name):
+        """
+        Gets the version specifier to use after parsing the project's requirements file.
+        
+        (See limitations and caveats in .requirements_parser.parse_requirements()
+        and .requirements_parser.get_version_specifier_for_pkg()).
+
+        If a requirements file isn't found at the root of the repo, or if there was an error parsing it,
+        a string representing that any version is allowed is returned. 
+        """
+        pkg_specifiers_map = {}
+        try:
+            pkg_specifiers_map = parse_requirements("requirements.txt", self.fs)
+        except (FileException, FileNotFoundError) as e:
+            log.warning("error parsing requirements file for {}, err: {}".format(self.PROJECT_ROOT, e))
+            pass
+
+        return get_version_specifier_for_pkg(package_name, pkg_specifiers_map)
+        
     def index_external_modules(self):
         for path in os.listdir(self.PACKAGES_PATH):
             if path not in self.indexed_folders:

langserver/workspace.py

@@ -0,0 +1,3 @@
+[packages]
+
+testfarhan = ">0.3,<0.5"

test/repos/dep_versioning/.DS_Store

@@ -0,0 +1 @@
+testfarhan>0.3,<0.5

test/repos/dep_versioning_between/Pipfile

@@ -0,0 +1,3 @@
+[packages]
+
+testfarhan = ">0.3,<0.6,!=0.5"

test/repos/dep_versioning_between/Pipfile.lock

@@ -0,0 +1 @@
+testfarhan>0.3,<0.6,!=0.5

test/repos/dep_versioning_between/requirements.txt

@@ -0,0 +1,3 @@
+from testfarhan.farhantest import testfunc
+
+print(testfunc())

test/repos/dep_versioning/test.py renamed to test/repos/dep_versioning_between/test.py

@@ -0,0 +1 @@
+testfarhan==0.1

test/repos/dep_versioning_between_multiple/Pipfile

@@ -0,0 +1,3 @@
+from testfarhan.farhantest import testfunc
+
+print(testfunc())

test/repos/dep_versioning_between_multiple/Pipfile.lock

@@ -0,0 +1,3 @@
+from testfarhan.farhantest import testfunc
+
+print(testfunc())

test/repos/dep_versioning_between_multiple/requirements.txt

@@ -3,22 +3,28 @@
 import pytest
 
 
-@pytest.fixture()
-def workspace():
-    workspace = Harness("repos/dep_versioning")
-    workspace.initialize("repos/dep_versioning?" + str(uuid.uuid4()))
-    yield workspace
-    workspace.exit()
+@pytest.fixture(params=[
+    # tuples of the repo for the test, along
+    # with the expected doc_string for the hover
+    # in that repo
+    ("repos/dep_versioning_fixed", "this is version 0.1"),
+    ("repos/dep_versioning_between", "this is version 0.4"),
+    ("repos/dep_versioning_between_multiple", "this is version 0.4"),
+    ("repos/dep_versioning_none", "this is version 0.6")
+])
+def test_data(request):
+    repo_path, expected_doc_string = request.param
 
+    workspace = Harness(repo_path)
+    workspace.initialize(repo_path + str(uuid.uuid4()))
+    yield (workspace, expected_doc_string)
 
-"""
-This test should pass as long as we do not fetch results from the correct version of dependencies .
-Once we *do*, this test should be updated so the hover returns 'this is version 0.1'.
-"""
+    workspace.exit()
 
 
 class TestDependencyVersioning:
-    def test_dep_version(self, workspace):
+    def test_dep_download_specified_version(self, test_data):
+        workspace, expected_doc_string = test_data
         uri = "file:///test.py"
         character, line = 6, 2
         result = workspace.hover(uri, line, character)
@@ -28,6 +34,6 @@ def test_dep_version(self, workspace):
                     'language': 'python',
                     'value': 'def testfunc()'
                 },
-                'this is version 0.2'
+                expected_doc_string
             ]
         }