diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b11da15..24493876 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2.0 +version: 2.1 filters: &filters filters: @@ -13,7 +13,6 @@ filters__tags: &filters__tags only: /^v[0-9]+(\.[0-9]+)*$/ workflows: - version: 2 test: jobs: - cache: @@ -34,19 +33,19 @@ workflows: <<: *filters requires: - cache - - test--9.6: + - test--9-6: <<: *filters requires: - cache - - test--9.5: + - test--9-5: <<: *filters requires: - cache - - test--9.4: + - test--9-4: <<: *filters requires: - cache - - test--8.4: + - test--8-4: <<: *filters requires: - cache @@ -58,11 +57,6 @@ workflows: <<: *filters requires: - test--12 - - test--11 - - test--10 - - test--9.6 - - test--9.5 - - test--9.4 - test-release: <<: *filters__tags requires: @@ -79,7 +73,7 @@ workflows: requires: - approve-release -cache: &cache deps-v6-{{ checksum "setup.py" }} +cache: &cache deps-v7-{{ checksum "setup.py" }} py: &py python:3.7.7-stretch @@ -88,10 +82,25 @@ restore__cache: &restore__cache keys: - *cache +# Simple checkout command to pull external forks. +# The CircleCI util does not work without setting up SSH keys +# which we technically do not need for open-source repos. +checkout_command: &checkout_command + run: + name: checkout + command: | + git clone https://github.com/datamill-co/target-postgres . + if [[ "$CIRCLE_BRANCH" =~ ^pull\/* ]]; then + git fetch origin refs/pull/${CIRCLE_PR_NUMBER}/head + git checkout ${CIRCLE_SHA1} + else + git checkout ${CIRCLE_BRANCH} + fi + test__base: &test__base working_directory: /code/ steps: - - checkout + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -117,7 +126,7 @@ jobs: docker: - image: *py steps: - - checkout + - *checkout_command - *restore__cache - run: @@ -137,7 +146,7 @@ jobs: command: | python -m venv venv/tap-github source venv/tap-github/bin/activate - pip install tap-github + pip install git+https://github.com/MeltanoLabs/tap-github.git@v1.1.0 deactivate - run: @@ -187,7 +196,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--9.6: + test--9-6: <<: *test__base docker: - image: *py @@ -196,7 +205,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--9.5: + test--9-5: <<: *test__base docker: - image: *py @@ -205,7 +214,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--9.4: + test--9-4: <<: *test__base docker: - image: *py @@ -214,7 +223,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--8.4: + test--8-4: <<: *test__base docker: - image: *py @@ -227,12 +236,12 @@ jobs: working_directory: /code/ docker: - image: *py - - image: postgres:9.4.26 + - image: postgres:12.2 environment: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres steps: - - checkout + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -252,7 +261,7 @@ jobs: source venv/tap-github/bin/activate cd /code/.circleci/integration/tap-github - tap-github --config config.json --properties properties.json > /code/artifacts/data/tap + tap-github --config config.json --catalog catalog.json > /code/artifacts/data/tap deactivate @@ -294,7 +303,7 @@ jobs: deactivate - cd /code + cd /code/ source venv/tap-postgres/bin/activate cd /code/.circleci/integration/tap-postgres @@ -324,7 +333,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres steps: - - checkout + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -349,7 +358,7 @@ jobs: docker: - image: *py steps: - - checkout + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -374,7 +383,7 @@ jobs: docker: - image: *py steps: - - checkout + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -404,7 +413,7 @@ jobs: docker: - image: *py steps: - - checkout + - *checkout_command - *restore__cache - attach_workspace: at: "./" diff --git a/.circleci/integration/tap-github/catalog.json b/.circleci/integration/tap-github/catalog.json new file mode 100644 index 00000000..6d55ec3e --- /dev/null +++ b/.circleci/integration/tap-github/catalog.json @@ -0,0 +1,1448 @@ +{ + "streams": [ + { + "tap_stream_id": "languages", + "replication_method": "FULL_TABLE", + "key_properties": [ + "repo", + "org", + "language_name" + ], + "schema": { + "properties": { + "repo": { + "type": [ + "string", + "null" + ] + }, + "org": { + "type": [ + "string", + "null" + ] + }, + "language_name": { + "type": [ + "string", + "null" + ] + }, + "bytes": { + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "stream": "languages", + "metadata": [ + { + "breadcrumb": [ + "properties", + "repo" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "org" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "language_name" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "bytes" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "selected": true, + "table-key-properties": [ + "repo", + "org", + "language_name" + ] + } + } + ] + }, + { + "tap_stream_id": "repositories", + "replication_key": "updated_at", + "replication_method": "INCREMENTAL", + "key_properties": [ + "id" + ], + "schema": { + "properties": { + "search_name": { + "type": [ + "string", + "null" + ] + }, + "search_query": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "repo": { + "type": [ + "string", + "null" + ] + }, + "org": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "full_name": { + "type": [ + "string", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "owner": { + "properties": { + "login": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "avatar_url": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "site_admin": { + "type": [ + "boolean", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "license": { + "properties": { + "key": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "spdx_id": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "master_branch": { + "type": [ + "string", + "null" + ] + }, + "default_branch": { + "type": [ + "string", + "null" + ] + }, + "updated_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "created_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "pushed_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "git_url": { + "type": [ + "string", + "null" + ] + }, + "ssh_url": { + "type": [ + "string", + "null" + ] + }, + "clone_url": { + "type": [ + "string", + "null" + ] + }, + "homepage": { + "type": [ + "string", + "null" + ] + }, + "private": { + "type": [ + "boolean", + "null" + ] + }, + "archived": { + "type": [ + "boolean", + "null" + ] + }, + "disabled": { + "type": [ + "boolean", + "null" + ] + }, + "size": { + "type": [ + "integer", + "null" + ] + }, + "stargazers_count": { + "type": [ + "integer", + "null" + ] + }, + "fork": { + "type": [ + "boolean", + "null" + ] + }, + "topics": { + "items": { + "type": [ + "string" + ] + }, + "type": [ + "array", + "null" + ] + }, + "visibility": { + "type": [ + "string", + "null" + ] + }, + "language": { + "type": [ + "string", + "null" + ] + }, + "forks": { + "type": [ + "integer", + "null" + ] + }, + "forks_count": { + "type": [ + "integer", + "null" + ] + }, + "watchers": { + "type": [ + "integer", + "null" + ] + }, + "watchers_count": { + "type": [ + "integer", + "null" + ] + }, + "open_issues": { + "type": [ + "integer", + "null" + ] + }, + "network_count": { + "type": [ + "integer", + "null" + ] + }, + "subscribers_count": { + "type": [ + "integer", + "null" + ] + }, + "open_issues_count": { + "type": [ + "integer", + "null" + ] + }, + "allow_squash_merge": { + "type": [ + "boolean", + "null" + ] + }, + "allow_merge_commit": { + "type": [ + "boolean", + "null" + ] + }, + "allow_rebase_merge": { + "type": [ + "boolean", + "null" + ] + }, + "allow_auto_merge": { + "type": [ + "boolean", + "null" + ] + }, + "delete_branch_on_merge": { + "type": [ + "boolean", + "null" + ] + }, + "organization": { + "properties": { + "login": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "avatar_url": { + "type": [ + "string", + "null" + ] + }, + "gravatar_id": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "site_admin": { + "type": [ + "boolean", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + } + }, + "type": "object" + }, + "stream": "repositories", + "metadata": [ + { + "breadcrumb": [ + "properties", + "search_name" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "search_query" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "id" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "node_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "repo" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "org" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "name" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "full_name" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "description" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "html_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "owner" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "license" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "master_branch" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "default_branch" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "updated_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "created_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "pushed_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "git_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "ssh_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "clone_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "homepage" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "private" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "archived" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "disabled" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "size" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "stargazers_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "fork" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "topics" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "visibility" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "language" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "forks" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "forks_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "watchers" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "watchers_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "open_issues" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "network_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "subscribers_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "open_issues_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_squash_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_merge_commit" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_rebase_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_auto_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "delete_branch_on_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "organization" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "selected": true, + "table-key-properties": [ + "id" + ], + "valid-replication-keys": [ + "updated_at" + ] + } + } + ] + }, + { + "tap_stream_id": "review_comments", + "replication_method": "FULL_TABLE", + "key_properties": [ + "id" + ], + "schema": { + "properties": { + "org": { + "type": [ + "string", + "null" + ] + }, + "repo": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "pull_request_review_id": { + "type": [ + "integer", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "diff_hunk": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": [ + "string", + "null" + ] + }, + "position": { + "type": [ + "integer", + "null" + ] + }, + "original_position": { + "type": [ + "integer", + "null" + ] + }, + "commit_id": { + "type": [ + "string", + "null" + ] + }, + "original_commit_id": { + "type": [ + "string", + "null" + ] + }, + "in_reply_to_id": { + "type": [ + "integer", + "null" + ] + }, + "user": { + "properties": { + "login": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "avatar_url": { + "type": [ + "string", + "null" + ] + }, + "gravatar_id": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "site_admin": { + "type": [ + "boolean", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "body": { + "type": [ + "string", + "null" + ] + }, + "created_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "updated_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "pull_request_url": { + "type": [ + "string", + "null" + ] + }, + "author_association": { + "type": [ + "string", + "null" + ] + }, + "_links": { + "properties": { + "self": { + "properties": { + "href": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "html": { + "properties": { + "href": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "pull_request": { + "properties": { + "href": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "start_line": { + "type": [ + "integer", + "null" + ] + }, + "original_start_line": { + "type": [ + "integer", + "null" + ] + }, + "start_side": { + "type": [ + "string", + "null" + ] + }, + "line": { + "type": [ + "integer", + "null" + ] + }, + "original_line": { + "type": [ + "integer", + "null" + ] + }, + "side": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "stream": "review_comments", + "metadata": [ + { + "breadcrumb": [ + "properties", + "org" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "repo" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "pull_request_review_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "id" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "node_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "diff_hunk" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "path" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "position" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_position" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "commit_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_commit_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "in_reply_to_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "user" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "body" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "created_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "updated_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "html_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "pull_request_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "author_association" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "_links" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "start_line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_start_line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "start_side" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "side" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "selected": true, + "table-key-properties": [ + "id" + ] + } + } + ] + } + ] +} diff --git a/.circleci/integration/tap-github/config-template.json b/.circleci/integration/tap-github/config-template.json index 5c7a9084..d9d9b625 100644 --- a/.circleci/integration/tap-github/config-template.json +++ b/.circleci/integration/tap-github/config-template.json @@ -1,4 +1,4 @@ { - "access_token": "REPLACE_ME", - "repository": "datamill-co/target-postgres" -} \ No newline at end of file + "start_date": "2022-04-12", + "repositories": ["datamill-co/target-postgres"] +} diff --git a/.circleci/integration/tap-github/properties.json b/.circleci/integration/tap-github/properties.json deleted file mode 100644 index 6f2812c5..00000000 --- a/.circleci/integration/tap-github/properties.json +++ /dev/null @@ -1,2414 +0,0 @@ -{ - "streams": [ - { - "stream": "stargazers", - "tap_stream_id": "stargazers", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "starred_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "user_id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "user_id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "starred_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user_id" - ], - "metadata": { - "inclusion": "automatic" - } - } - ], - "key_properties": [ - "user_id" - ] - }, - { - "stream": "assignees", - "tap_stream_id": "assignees", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "_sdc_repository": { - "type": [ - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "login" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "type" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "reviews", - "tap_stream_id": "reviews", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "commit_id": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "pull_request_url": { - "type": [ - "null", - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "state" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commit_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request_url" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "issues", - "tap_stream_id": "issues", - "schema": { - "selected": true, - "properties": { - "state": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "labels": { - "type": [ - "null", - "array" - ], - "items": { - "type": "object", - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "string" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - } - } - } - }, - "repository_url": { - "type": [ - "null", - "string" - ] - }, - "number": { - "type": [ - "null", - "integer" - ] - }, - "labels_url": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "assignee": { - "type": [ - "null", - "object" - ], - "properties": {} - }, - "updated_at": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "author_association": { - "type": [ - "null", - "string" - ] - }, - "locked": { - "type": [ - "null", - "boolean" - ] - }, - "events_url": { - "type": [ - "null", - "string" - ] - }, - "pull_request": { - "properties": { - "diff_url": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "patch_url": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "comments": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ] - }, - "_sdc_repository": { - "type": [ - "string" - ] - }, - "user": { - "properties": { - "repos_url": { - "type": [ - "null", - "string" - ] - }, - "starred_url": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "received_events_url": { - "type": [ - "null", - "string" - ] - }, - "site_admin": { - "type": [ - "null", - "boolean" - ] - }, - "gravatar_id": { - "type": [ - "null", - "string" - ] - }, - "following_url": { - "type": [ - "null", - "string" - ] - }, - "avatar_url": { - "type": [ - "null", - "string" - ] - }, - "events_url": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "login": { - "type": [ - "null", - "string" - ] - }, - "organizations_url": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "subscriptions_url": { - "type": [ - "null", - "string" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "followers_url": { - "type": [ - "null", - "string" - ] - }, - "gists_url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "comments_url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "state" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "labels" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "repository_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "number" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "closed_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "labels_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "title" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "assignee" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "author_association" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "locked" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "events_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "node_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "comments" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "milestone" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "review_comments", - "tap_stream_id": "review_comments", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "pull_request_review_id": { - "type": [ - "null", - "integer" - ] - }, - "diff_hunk": { - "type": [ - "null", - "string" - ] - }, - "path": { - "type": [ - "null", - "string" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "original_position": { - "type": [ - "null", - "integer" - ] - }, - "commit_id": { - "type": [ - "null", - "string" - ] - }, - "original_commit_id": { - "type": [ - "null", - "string" - ] - }, - "in_reply_to_id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "pull_request_url": { - "type": [ - "null", - "string" - ] - }, - "assignee": { - "type": [ - "null", - "string" - ] - }, - "assignees": { - "type": [ - "null", - "string" - ] - }, - "author_association": { - "type": [ - "null", - "string" - ] - }, - "base": { - "type": [ - "null", - "string" - ] - }, - "comments_url": { - "type": [ - "null", - "string" - ] - }, - "commits_url": { - "type": [ - "null", - "string" - ] - }, - "diff_url": { - "type": [ - "null", - "string" - ] - }, - "head": { - "type": [ - "null", - "string" - ] - }, - "issue_url": { - "type": [ - "null", - "string" - ] - }, - "labels": { - "type": [ - "null", - "string" - ] - }, - "locked": { - "type": [ - "null", - "string" - ] - }, - "merge_commit_sha": { - "type": [ - "null", - "string" - ] - }, - "milestone": { - "type": [ - "null", - "string" - ] - }, - "patch_url": { - "type": [ - "null", - "string" - ] - }, - "requested_reviewers": { - "type": [ - "null", - "string" - ] - }, - "requested_teams": { - "type": [ - "null", - "string" - ] - }, - "review_comment_url": { - "type": [ - "null", - "string" - ] - }, - "review_comments_url": { - "type": [ - "null", - "string" - ] - }, - "statuses_url": { - "type": [ - "null", - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "node_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request_review_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "diff_hunk" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "path" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "position" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "original_position" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commit_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "original_commit_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "in_reply_to_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "assignee" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "assignees" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "author_association" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "base" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commits_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "diff_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "head" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "issue_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "labels" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "locked" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "merge_commit_sha" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "milestone" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "patch_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "requested_reviewers" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "requested_teams" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "review_comment_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "review_comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "statuses_url" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "collaborators", - "tap_stream_id": "collaborators", - "schema": { - "selected": false, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "_sdc_repository": { - "type": [ - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "login" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "type" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "comments", - "tap_stream_id": "comments", - "schema": { - "selected": true, - "type": "object", - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "home_url": { - "type": [ - "null", - "string" - ] - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "issue_url": { - "type": [ - "null", - "string" - ] - }, - "author_association": { - "type": [ - "null", - "string" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "avatar_url": { - "type": [ - "null", - "string" - ] - }, - "gravatar_id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "followers_url": { - "type": [ - "null", - "string" - ] - }, - "following_url": { - "type": [ - "null", - "string" - ] - }, - "gists_url": { - "type": [ - "null", - "string" - ] - }, - "starred_url": { - "type": [ - "null", - "string" - ] - }, - "subscriptions_url": { - "type": [ - "null", - "string" - ] - }, - "organizations_url": { - "type": [ - "null", - "string" - ] - }, - "repos_url": { - "type": [ - "null", - "string" - ] - }, - "events_url": { - "type": [ - "null", - "string" - ] - }, - "received_events_url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "site_admin": { - "type": [ - "null", - "string" - ] - } - } - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "_sdc_repository": { - "type": [ - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "node_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "home_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "issue_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "author_association" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "pull_requests", - "tap_stream_id": "pull_requests", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "number": { - "type": [ - "null", - "integer" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "merged_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "closed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "number" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "state" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "title" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "merged_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "closed_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "commits", - "tap_stream_id": "commits", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "sha": { - "type": [ - "null", - "string" - ], - "description": "The git commit hash" - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "parents": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "sha": { - "type": [ - "null", - "string" - ], - "description": "The git hash of the parent commit" - }, - "url": { - "type": [ - "null", - "string" - ], - "description": "The URL to the parent commit" - }, - "html_url": { - "type": [ - "null", - "string" - ], - "description": "The HTML URL to the parent commit" - } - } - } - }, - "html_url": { - "type": [ - "null", - "string" - ], - "description": "The HTML URL to the commit" - }, - "comments_url": { - "type": [ - "null", - "string" - ], - "description": "The URL to the commit's comments page" - }, - "commit": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "url": { - "type": [ - "null", - "string" - ], - "description": "The URL to the commit" - }, - "tree": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "sha": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - } - } - }, - "author": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "date": { - "type": [ - "null", - "string" - ], - "format": "date-time", - "description": "The date the author committed the change" - }, - "name": { - "type": [ - "null", - "string" - ], - "description": "The author's name" - }, - "email": { - "type": [ - "null", - "string" - ], - "description": "The author's email" - } - } - }, - "message": { - "type": [ - "null", - "string" - ], - "description": "The commit message" - }, - "committer": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "date": { - "type": [ - "null", - "string" - ], - "format": "date-time", - "description": "The date the committer committed the change" - }, - "name": { - "type": [ - "null", - "string" - ], - "description": "The committer's name" - }, - "email": { - "type": [ - "null", - "string" - ], - "description": "The committer's email" - } - } - }, - "comment_count": { - "type": [ - "null", - "integer" - ], - "description": "The number of comments on the commit" - } - } - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "sha" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "sha" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "parents" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commit" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "sha" - ] - } - ] -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..e59e208c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contribution guidelines + +## Setting up test environment + +### Prerequisits + +``` +python3 -m virtualenv python3 venv +source venv/bin/activate +python3 -m pip install -e .[tests] +# python3 -m pip install -e .\[tests\] <- night need to escape on zsh +export POSTGRES_HOST=localhost +export POSTGRES_DATABASE=target_postgres_test +export POSTGRES_USERNAME=target_postgres_test +export POSTGRES_PASSWORD=target_postgres_test +``` + +#### Database setup +If you're not using the docker images for tests you'll need to set one up and +configure a user on it. + +``` +$ psql template1; +<>=# CREATE USER target_postgres_test WITH PASSWORD 'target_postgres_test'; +<>=# CREATE DATABASE target_postgres_test WITH owner=target_postgres_test; +<>=# GRANT ALL privileges ON DATABASE target_postgres_test TO target_postgres_test; +``` + +#### If psycopg2 install fails + +psycopg2 requires ssl and may fail the `pip install` process above + +##### Installing openssl + +###### OSX: + +One possible solution is to use [homebrew](https://brew.sh/): + +``` +brew install openssl@1.1 +export LDFLAGS="-L/usr/local/opt/openssl@1.1/lib" +export CPPFLAGS="-I/usr/local/opt/openssl@1.1/include +python3 -m pip install -r requirements_test.pip +``` + +## Running tests +Tests are written using [pytest](https://docs.pytest.org/). + +``` +cd +python3 -m pytest tests/unit +``` + +Simply run the tests with pytest as a module when inside the root of the +checkout; this ensures the `target_postgres/` module directory is found on the +`PYTHONPATH`. diff --git a/README.md b/README.md index 6e344184..cd2baf62 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ here. | `add_upsert_indexes` | `["boolean", "null"]` | `True` | Whether the Target should create column indexes on the important columns used during data loading. These indexes will make data loading slightly slower but the deduplication phase much faster. Defaults to on for better baseline performance. | | `before_run_sql` | `["string", "null"]` | `None` | Raw SQL statement(s) to execute as soon as the connection to Postgres is opened by the target. Useful for setup like `SET ROLE` or other connection state that is important. | | `after_run_sql` | `["string", "null"]` | `None` | Raw SQL statement(s) to execute as soon as the connection to Postgres is opened by the target. Useful for setup like `SET ROLE` or other connection state that is important. | +| `before_run_sql_file` | `["string", "null"]` | `None` | Similar to `before_run_sql` but reads an external file instead of SQL in the JSON config file. | +| `after_run_sql_file` | `["string", "null"]` | `None` | Similar to `after_run_sql` but reads an external file instead of SQL in the JSON config file. | ### Supported Versions diff --git a/target_postgres/postgres.py b/target_postgres/postgres.py index 97d9ccf0..223c0d55 100644 --- a/target_postgres/postgres.py +++ b/target_postgres/postgres.py @@ -1,5 +1,6 @@ from copy import deepcopy import csv +from functools import lru_cache import io import json import logging @@ -19,6 +20,14 @@ RESERVED_NULL_DEFAULT = 'NULL' +@lru_cache(maxsize=128) +def _format_datetime(value): + """ + Format a datetime value. This is only called from the + PostgresTarget.serialize_table_record_datetime_value + but this non-method version allows caching + """ + return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ') def _update_schema_0_to_1(table_metadata, table_schema): """ @@ -546,7 +555,7 @@ def serialize_table_record_null_value(self, remote_schema, streamed_schema, fiel return value def serialize_table_record_datetime_value(self, remote_schema, streamed_schema, field, value): - return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ') + return _format_datetime(value) def persist_csv_rows(self, cur, diff --git a/target_postgres/sql_base.py b/target_postgres/sql_base.py index 9d879526..2064005d 100644 --- a/target_postgres/sql_base.py +++ b/target_postgres/sql_base.py @@ -13,6 +13,8 @@ # from copy import deepcopy +from functools import lru_cache, partial +import pickle import time import singer @@ -628,16 +630,23 @@ def log_message(msg): return self._get_table_schema(connection, table_name) - def _serialize_table_record_field_name(self, remote_schema, path, value_json_schema): + def _serialize_table_record_field_name(self, remote_schema, path, value_json_schema_tuple): """ Returns the appropriate remote field (column) name for `path`. :param remote_schema: TABLE_SCHEMA(remote) :param path: (string, ...) - :value_json_schema: dict, JSON Schema + :value_json_schema: tuple, JSON Schema :return: string """ + # rebuild the dict that needs to be passed further down the call stack + if len(value_json_schema_tuple) == 1: + value_json_schema = { 'type': value_json_schema_tuple[0] } + else: + value_json_schema = {'type': value_json_schema_tuple[0], + 'format': value_json_schema_tuple[1]} + simple_json_schema = json_schema.simple_type(value_json_schema) mapping = self._get_mapping(remote_schema, @@ -725,9 +734,15 @@ def _serialize_table_records( default_row = dict([(field, NULL_DEFAULT) for field in remote_fields]) paths = streamed_schema['schema']['properties'].keys() + + ## create a partial function with only hashable args so we can use lru_cache on it + _cached_field_name = partial(self._serialize_table_record_field_name, remote_schema) + cached_field_name = lru_cache(maxsize=None)(_cached_field_name) + for record in records: - row = deepcopy(default_row) + ## pickling/unpickling is much faster than deepcopy + row = pickle.loads(pickle.dumps(default_row)) for path in paths: json_schema_string_type, value = record.get(path, (None, None)) @@ -747,18 +762,14 @@ def _serialize_table_records( and value is not None: value = self.serialize_table_record_datetime_value(remote_schema, streamed_schema, path, value) - value_json_schema = {'type': json_schema.STRING, - 'format': json_schema.DATE_TIME_FORMAT} + value_json_schema_tuple = (json_schema.STRING, json_schema.DATE_TIME_FORMAT) + field_name = cached_field_name(path, value_json_schema_tuple) else: - value_json_schema = {'type': json_schema_string_type} + field_name = cached_field_name(path, (json_schema_string_type,)) ## Serialize NULL default value value = self.serialize_table_record_null_value(remote_schema, streamed_schema, path, value) - field_name = self._serialize_table_record_field_name(remote_schema, - path, - value_json_schema) - ## `field_name` is unset if row[field_name] == NULL_DEFAULT: row[field_name] = value diff --git a/target_postgres/stream_tracker.py b/target_postgres/stream_tracker.py index 7e68a653..e2b12b88 100644 --- a/target_postgres/stream_tracker.py +++ b/target_postgres/stream_tracker.py @@ -48,9 +48,9 @@ def flush_streams(self, force=False): self._emit_safe_queued_states(force=force) - def handle_state_message(self, line_data): + def handle_state_message(self, line): if self.emit_states: - self.state_queue.append({'state': line_data['value'], 'watermark': self.message_counter}) + self.state_queue.append({'state': line, 'watermark': self.message_counter}) self._emit_safe_queued_states() def handle_record_message(self, stream, line_data): @@ -80,9 +80,14 @@ def _emit_safe_queued_states(self, force=False): valid_flush_watermarks.append(watermark) safe_flush_threshold = min(valid_flush_watermarks, default=0) + # the STATE message that the target forwards emittable_state = None + emittable_state_str = None while len(self.state_queue) > 0 and (force or self.state_queue[0]['watermark'] <= safe_flush_threshold): - emittable_state = self.state_queue.popleft()['state'] + emittable_state_str = self.state_queue.popleft()['state'] + + if emittable_state_str is not None: + emittable_state = json.loads(emittable_state_str)['value'] if emittable_state: if len(statediff.diff(emittable_state, self.last_emitted_state or {})) > 0: diff --git a/target_postgres/target_tools.py b/target_postgres/target_tools.py index c01cc40b..197babcd 100644 --- a/target_postgres/target_tools.py +++ b/target_postgres/target_tools.py @@ -152,7 +152,8 @@ def _line_handler(state_tracker, target, invalid_records_detect, invalid_records state_tracker.flush_stream(line_data['stream']) target.activate_version(stream_buffer, line_data['version']) elif line_data['type'] == 'STATE': - state_tracker.handle_state_message(line_data) + # pass the string instead of the deserialized object to save memory in the deque + state_tracker.handle_state_message(line) else: raise TargetError('Unknown message type {} in message {}'.format( line_data['type'], @@ -188,3 +189,10 @@ def _run_sql_hook(hook_name, config, target): with target.conn.cursor() as cur: cur.execute(config[hook_name]) LOGGER.debug('{} SQL executed'.format(hook_name)) + + hook_file = hook_name + '_file' + if hook_file in config: + with open(config[hook_file]) as f: + with target.conn.cursor() as cur: + cur.execute(f.read()) + LOGGER.debug('{} SQL file executed'.format(hook_file))