Poetry for Dependency Management and TestContainers for Integration Testing (#134)

In this PR:
- Uses Poetry for Dependency Management
- Replaces the Makefile with Python scripts that can be run from Poetry
- Implements TestContainers with a Docker Compose file for integration
testing

---------

Co-authored-by: Robert Shelton <[email protected]>
Co-authored-by: Tyler Hutcherson <[email protected]>

Author: 3 people

.github/workflows/lint.yml

@@ -21,7 +21,7 @@ jobs:
         # Starting new jobs is also relatively slow,
         # so linting on fewer versions makes CI faster.
         python-version:
-          - "3.8"
+          - "3.9"
           - "3.11"
 
     steps:
@@ -30,19 +30,17 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install .[dev,all]
-
+        poetry install --all-extras
     - name: check-sort-import
       run: |
-        make check-sort-imports
-
+        poetry run check-sort-imports
     - name: check-black-format
       run: |
-        make check-format
-
+        poetry run check-format
     - name: check-mypy
       run: |
-        make mypy
+        poetry run mypy

.github/workflows/publish.yml

@@ -18,13 +18,15 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install setuptools wheel
+          poetry install --all-extras
 
-      - name: create python package
-        run: python setup.py sdist bdist_wheel
+      - name: Build package
+        run: poetry build
 
       - name: Create Release
         uses: ncipollo/release-action@v1
@@ -36,8 +38,7 @@ jobs:
           tag: ${{ github.event.inputs.version }}
           commit: main
 
-      - name: deploy
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI }}
+      - name: Publish to PyPI
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI }}
+        run: poetry publish

.github/workflows/run_tests.yml

@@ -15,9 +15,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: [3.9, '3.10', 3.11]
         connection: ['hiredis', 'plain']
-        redis-stack-version: ['6.2.6-v9', 'latest']
+        redis-stack-version: ['6.2.6-v9', 'latest', 'edge']
 
     services:
       redis:
@@ -33,20 +33,21 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
 
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install .[dev,all]
+        poetry install --all-extras
 
     - name: Install hiredis if needed
       if: matrix.connection == 'hiredis'
       run: |
-        pip install hiredis
+        poetry add hiredis
 
-    - name: Start Redis
+    - name: Set Redis version
       run: |
-        REDIS_URL=redis://localhost:6379
-        echo REDIS_URL=$REDIS_URL >> $GITHUB_ENV
+        echo "REDIS_VERSION=${{ matrix.redis-stack-version }}" >> $GITHUB_ENV
 
     - name: Authenticate to Google Cloud
       uses: google-github-actions/auth@v1
@@ -65,12 +66,12 @@ jobs:
         AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}}
         OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}}
       run: |
-        make test-cov
+        poetry run test-cov
 
     - name: Run tests
       if: matrix.connection != 'plain' || matrix.redis-stack-version != 'latest'
       run: |
-        SKIP_VECTORIZERS=True make test-cov
+        SKIP_VECTORIZERS=True poetry run test-cov
 
     - name: Run notebooks
       if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest'
@@ -84,10 +85,11 @@ jobs:
         AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}}
         OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}}
       run: |
-        cd docs/ && treon -v --exclude="./examples/openai_qna.ipynb"
+        cd docs/ && poetry run treon -v --exclude="./examples/openai_qna.ipynb"
 
     - name: Publish coverage results
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v4
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        fail_ci_if_error: true
+        files: ./coverage.xml
+        fail_ci_if_error: true

.github/workflows/test-publish.yml

@@ -11,20 +11,22 @@ jobs:
   build_release:
     name: Build Release
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install setuptools wheel
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
 
-      - name: create python package
-        run: python setup.py sdist bdist_wheel
+      - name: Build package
+        run: |
+          poetry version ${{ github.event.inputs.version }}
+          poetry build
 
       - name: Create Release
         uses: ncipollo/release-action@v1
@@ -36,9 +38,9 @@ jobs:
           tag: ${{ github.event.inputs.version }}
           commit: main
 
-      - name: deploy
+      - name: Publish to TestPyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
-          password: ${{ secrets.PYPI }}
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository_url: https://test.pypi.org/legacy/

.gitignore

@@ -2,9 +2,11 @@ __pycache__/
 redisvl.egg-info/
 .coverage.*
 .coverage
+coverage.xml
 scratch
 .DS_Store
 *.csv
 wiki_schema.yaml
 docs/_build/
-.venv
+.venv
+coverage.xml

CONTRIBUTING.md

@@ -28,45 +28,49 @@ Here's how to get started with your code contribution:
     pull request.
 
 ### Dev Environment
-There is a provided `requirements.txt` and `requirements-dev.txt` file you can use to install required libraries with `pip` into your virtual environment.
+RedisVL uses [Poetry](https://python-poetry.org/) for dependency management.
+
+Follow the instructions to [install Poetry](https://python-poetry.org/docs/#installation).
+
+Then install the required libraries:
 
-Or use the local package editable install method:
 ```bash
-python -m venv .venv
-source .venv/bin/activate
-pip install -e '.[all,dev]'
+poetry install --all-extras
 ```
 
-Then to deactivate the env:
-```
-source deactivate
-```
 
 ### Linting and Tests
 
 Check formatting, linting, and typing:
 ```bash
-make check
+poetry run format
+poetry run sort-imports
+poetry run mypy
 ```
 
+#### TestContainers
+
+RedisVL uses Testcontainers Python for integration tests. Testcontainers is an open-source framework for provisioning throwaway, on-demand containers for development and testing use cases.
+
+To run Testcontainers-based tests you need a local Docker installation such as:
+- [Docker Desktop](https://www.docker.com/products/docker-desktop/)
+- [Docker Engine on Linux](https://docs.docker.com/engine/install/)
+
+#### Running the Tests
+
 Tests (with vectorizers):
 ```bash
-make test-cov
+poetry run test-cov
 ```
 
-**NOTE**: Some tests require the `REDIS_URL` environment variable to be set (e.g. `export REDIS_URL=redis://localhost:6379`).
-
 Tests w/out vectorizers:
 ```bash
-SKIP_VECTORIZERS=true make test-cov
+SKIP_VECTORIZERS=true poetry run test-cov
 ```
 
-> Dev requirements are needed here to be able to run tests and linting.
-> See other commands in the [Makefile](Makefile)
-
-### Docker Tips
+### Getting Redis
 
-Make sure to have [Redis](https://redis.io) accessible with Search & Query features enabled on [Redis Cloud](https://redis.com/try-free) or locally in docker with [Redis Stack](https://redis.io/docs/getting-started/install-stack/docker/):
+In order for your applications to use RedisVL, you must have [Redis](https://redis.io) accessible with Search & Query features enabled on [Redis Cloud](https://redis.com/try-free) or locally in docker with [Redis Stack](https://redis.io/docs/getting-started/install-stack/docker/):
 
 ```bash
 docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest