From e145b28f0b0efeca9341f6f05e57a8ce371d00de Mon Sep 17 00:00:00 2001 From: Frank Harrison Date: Thu, 4 Mar 2021 09:39:01 +0000 Subject: [PATCH 1/5] Adds contribution docs to understand how to run the tests This only documents `tests/unit` as `tests/migrations` appear to be designed to work inside a container rather than a local dev machine. --- CONTRIBUTING.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e59e208 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contribution guidelines + +## Setting up test environment + +### Prerequisits + +``` +python3 -m virtualenv python3 venv +source venv/bin/activate +python3 -m pip install -e .[tests] +# python3 -m pip install -e .\[tests\] <- night need to escape on zsh +export POSTGRES_HOST=localhost +export POSTGRES_DATABASE=target_postgres_test +export POSTGRES_USERNAME=target_postgres_test +export POSTGRES_PASSWORD=target_postgres_test +``` + +#### Database setup +If you're not using the docker images for tests you'll need to set one up and +configure a user on it. + +``` +$ psql template1; +<>=# CREATE USER target_postgres_test WITH PASSWORD 'target_postgres_test'; +<>=# CREATE DATABASE target_postgres_test WITH owner=target_postgres_test; +<>=# GRANT ALL privileges ON DATABASE target_postgres_test TO target_postgres_test; +``` + +#### If psycopg2 install fails + +psycopg2 requires ssl and may fail the `pip install` process above + +##### Installing openssl + +###### OSX: + +One possible solution is to use [homebrew](https://brew.sh/): + +``` +brew install openssl@1.1 +export LDFLAGS="-L/usr/local/opt/openssl@1.1/lib" +export CPPFLAGS="-I/usr/local/opt/openssl@1.1/include +python3 -m pip install -r requirements_test.pip +``` + +## Running tests +Tests are written using [pytest](https://docs.pytest.org/). + +``` +cd +python3 -m pytest tests/unit +``` + +Simply run the tests with pytest as a module when inside the root of the +checkout; this ensures the `target_postgres/` module directory is found on the +`PYTHONPATH`. From 5c0e666770382f808820fcc52db3f4818a52b87c Mon Sep 17 00:00:00 2001 From: Laurent Savaete Date: Tue, 8 Feb 2022 00:51:48 +0000 Subject: [PATCH 2/5] Reduce memory footprint due to storing state messages --- target_postgres/stream_tracker.py | 11 ++++++++--- target_postgres/target_tools.py | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/target_postgres/stream_tracker.py b/target_postgres/stream_tracker.py index 7e68a65..e2b12b8 100644 --- a/target_postgres/stream_tracker.py +++ b/target_postgres/stream_tracker.py @@ -48,9 +48,9 @@ def flush_streams(self, force=False): self._emit_safe_queued_states(force=force) - def handle_state_message(self, line_data): + def handle_state_message(self, line): if self.emit_states: - self.state_queue.append({'state': line_data['value'], 'watermark': self.message_counter}) + self.state_queue.append({'state': line, 'watermark': self.message_counter}) self._emit_safe_queued_states() def handle_record_message(self, stream, line_data): @@ -80,9 +80,14 @@ def _emit_safe_queued_states(self, force=False): valid_flush_watermarks.append(watermark) safe_flush_threshold = min(valid_flush_watermarks, default=0) + # the STATE message that the target forwards emittable_state = None + emittable_state_str = None while len(self.state_queue) > 0 and (force or self.state_queue[0]['watermark'] <= safe_flush_threshold): - emittable_state = self.state_queue.popleft()['state'] + emittable_state_str = self.state_queue.popleft()['state'] + + if emittable_state_str is not None: + emittable_state = json.loads(emittable_state_str)['value'] if emittable_state: if len(statediff.diff(emittable_state, self.last_emitted_state or {})) > 0: diff --git a/target_postgres/target_tools.py b/target_postgres/target_tools.py index c01cc40..70fc060 100644 --- a/target_postgres/target_tools.py +++ b/target_postgres/target_tools.py @@ -152,7 +152,8 @@ def _line_handler(state_tracker, target, invalid_records_detect, invalid_records state_tracker.flush_stream(line_data['stream']) target.activate_version(stream_buffer, line_data['version']) elif line_data['type'] == 'STATE': - state_tracker.handle_state_message(line_data) + # pass the string instead of the deserialized object to save memory in the deque + state_tracker.handle_state_message(line) else: raise TargetError('Unknown message type {} in message {}'.format( line_data['type'], From 6a3da026d2bb4681fdf46bd7ca69fbb164489d8a Mon Sep 17 00:00:00 2001 From: Laurent Savaete Date: Fri, 8 Apr 2022 18:52:18 -0300 Subject: [PATCH 3/5] Multiple performance improvements (#204) --- target_postgres/postgres.py | 11 ++++++++++- target_postgres/sql_base.py | 31 +++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/target_postgres/postgres.py b/target_postgres/postgres.py index 97d9ccf..223c0d5 100644 --- a/target_postgres/postgres.py +++ b/target_postgres/postgres.py @@ -1,5 +1,6 @@ from copy import deepcopy import csv +from functools import lru_cache import io import json import logging @@ -19,6 +20,14 @@ RESERVED_NULL_DEFAULT = 'NULL' +@lru_cache(maxsize=128) +def _format_datetime(value): + """ + Format a datetime value. This is only called from the + PostgresTarget.serialize_table_record_datetime_value + but this non-method version allows caching + """ + return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ') def _update_schema_0_to_1(table_metadata, table_schema): """ @@ -546,7 +555,7 @@ def serialize_table_record_null_value(self, remote_schema, streamed_schema, fiel return value def serialize_table_record_datetime_value(self, remote_schema, streamed_schema, field, value): - return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ') + return _format_datetime(value) def persist_csv_rows(self, cur, diff --git a/target_postgres/sql_base.py b/target_postgres/sql_base.py index 9d87952..2064005 100644 --- a/target_postgres/sql_base.py +++ b/target_postgres/sql_base.py @@ -13,6 +13,8 @@ # from copy import deepcopy +from functools import lru_cache, partial +import pickle import time import singer @@ -628,16 +630,23 @@ def log_message(msg): return self._get_table_schema(connection, table_name) - def _serialize_table_record_field_name(self, remote_schema, path, value_json_schema): + def _serialize_table_record_field_name(self, remote_schema, path, value_json_schema_tuple): """ Returns the appropriate remote field (column) name for `path`. :param remote_schema: TABLE_SCHEMA(remote) :param path: (string, ...) - :value_json_schema: dict, JSON Schema + :value_json_schema: tuple, JSON Schema :return: string """ + # rebuild the dict that needs to be passed further down the call stack + if len(value_json_schema_tuple) == 1: + value_json_schema = { 'type': value_json_schema_tuple[0] } + else: + value_json_schema = {'type': value_json_schema_tuple[0], + 'format': value_json_schema_tuple[1]} + simple_json_schema = json_schema.simple_type(value_json_schema) mapping = self._get_mapping(remote_schema, @@ -725,9 +734,15 @@ def _serialize_table_records( default_row = dict([(field, NULL_DEFAULT) for field in remote_fields]) paths = streamed_schema['schema']['properties'].keys() + + ## create a partial function with only hashable args so we can use lru_cache on it + _cached_field_name = partial(self._serialize_table_record_field_name, remote_schema) + cached_field_name = lru_cache(maxsize=None)(_cached_field_name) + for record in records: - row = deepcopy(default_row) + ## pickling/unpickling is much faster than deepcopy + row = pickle.loads(pickle.dumps(default_row)) for path in paths: json_schema_string_type, value = record.get(path, (None, None)) @@ -747,18 +762,14 @@ def _serialize_table_records( and value is not None: value = self.serialize_table_record_datetime_value(remote_schema, streamed_schema, path, value) - value_json_schema = {'type': json_schema.STRING, - 'format': json_schema.DATE_TIME_FORMAT} + value_json_schema_tuple = (json_schema.STRING, json_schema.DATE_TIME_FORMAT) + field_name = cached_field_name(path, value_json_schema_tuple) else: - value_json_schema = {'type': json_schema_string_type} + field_name = cached_field_name(path, (json_schema_string_type,)) ## Serialize NULL default value value = self.serialize_table_record_null_value(remote_schema, streamed_schema, path, value) - field_name = self._serialize_table_record_field_name(remote_schema, - path, - value_json_schema) - ## `field_name` is unset if row[field_name] == NULL_DEFAULT: row[field_name] = value From c0b0e0d191e174f116010e931ea923c00ef03b3b Mon Sep 17 00:00:00 2001 From: Eric Boucher Date: Thu, 14 Apr 2022 18:17:44 -0400 Subject: [PATCH 4/5] Fix circleCI (bypass need for SSH key) (#213) --- .circleci/config.yml | 48 +- .circleci/integration/tap-github/catalog.json | 1448 ++++++++++ .../tap-github/config-template.json | 6 +- .../integration/tap-github/properties.json | 2414 ----------------- 4 files changed, 1472 insertions(+), 2444 deletions(-) create mode 100644 .circleci/integration/tap-github/catalog.json delete mode 100644 .circleci/integration/tap-github/properties.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b11da1..35fa136 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2.0 +version: 2.1 filters: &filters filters: @@ -13,7 +13,6 @@ filters__tags: &filters__tags only: /^v[0-9]+(\.[0-9]+)*$/ workflows: - version: 2 test: jobs: - cache: @@ -34,19 +33,19 @@ workflows: <<: *filters requires: - cache - - test--9.6: + - test--9-6: <<: *filters requires: - cache - - test--9.5: + - test--9-5: <<: *filters requires: - cache - - test--9.4: + - test--9-4: <<: *filters requires: - cache - - test--8.4: + - test--8-4: <<: *filters requires: - cache @@ -58,11 +57,6 @@ workflows: <<: *filters requires: - test--12 - - test--11 - - test--10 - - test--9.6 - - test--9.5 - - test--9.4 - test-release: <<: *filters__tags requires: @@ -79,7 +73,7 @@ workflows: requires: - approve-release -cache: &cache deps-v6-{{ checksum "setup.py" }} +cache: &cache deps-v7-{{ checksum "setup.py" }} py: &py python:3.7.7-stretch @@ -91,7 +85,7 @@ restore__cache: &restore__cache test__base: &test__base working_directory: /code/ steps: - - checkout + - run: git clone https://github.com/datamill-co/target-postgres . - *restore__cache - attach_workspace: at: "./" @@ -117,7 +111,7 @@ jobs: docker: - image: *py steps: - - checkout + - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} - *restore__cache - run: @@ -137,7 +131,7 @@ jobs: command: | python -m venv venv/tap-github source venv/tap-github/bin/activate - pip install tap-github + pip install git+https://github.com/MeltanoLabs/tap-github.git@v1.1.0 deactivate - run: @@ -187,7 +181,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--9.6: + test--9-6: <<: *test__base docker: - image: *py @@ -196,7 +190,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--9.5: + test--9-5: <<: *test__base docker: - image: *py @@ -205,7 +199,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--9.4: + test--9-4: <<: *test__base docker: - image: *py @@ -214,7 +208,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres - test--8.4: + test--8-4: <<: *test__base docker: - image: *py @@ -227,12 +221,12 @@ jobs: working_directory: /code/ docker: - image: *py - - image: postgres:9.4.26 + - image: postgres:12.2 environment: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres steps: - - checkout + - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} - *restore__cache - attach_workspace: at: "./" @@ -252,7 +246,7 @@ jobs: source venv/tap-github/bin/activate cd /code/.circleci/integration/tap-github - tap-github --config config.json --properties properties.json > /code/artifacts/data/tap + tap-github --config config.json --catalog catalog.json > /code/artifacts/data/tap deactivate @@ -294,7 +288,7 @@ jobs: deactivate - cd /code + cd /code/ source venv/tap-postgres/bin/activate cd /code/.circleci/integration/tap-postgres @@ -324,7 +318,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres steps: - - checkout + - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} - *restore__cache - attach_workspace: at: "./" @@ -349,7 +343,7 @@ jobs: docker: - image: *py steps: - - checkout + - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} - *restore__cache - attach_workspace: at: "./" @@ -374,7 +368,7 @@ jobs: docker: - image: *py steps: - - checkout + - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} - *restore__cache - attach_workspace: at: "./" @@ -404,7 +398,7 @@ jobs: docker: - image: *py steps: - - checkout + - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} - *restore__cache - attach_workspace: at: "./" diff --git a/.circleci/integration/tap-github/catalog.json b/.circleci/integration/tap-github/catalog.json new file mode 100644 index 0000000..6d55ec3 --- /dev/null +++ b/.circleci/integration/tap-github/catalog.json @@ -0,0 +1,1448 @@ +{ + "streams": [ + { + "tap_stream_id": "languages", + "replication_method": "FULL_TABLE", + "key_properties": [ + "repo", + "org", + "language_name" + ], + "schema": { + "properties": { + "repo": { + "type": [ + "string", + "null" + ] + }, + "org": { + "type": [ + "string", + "null" + ] + }, + "language_name": { + "type": [ + "string", + "null" + ] + }, + "bytes": { + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "stream": "languages", + "metadata": [ + { + "breadcrumb": [ + "properties", + "repo" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "org" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "language_name" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "bytes" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "selected": true, + "table-key-properties": [ + "repo", + "org", + "language_name" + ] + } + } + ] + }, + { + "tap_stream_id": "repositories", + "replication_key": "updated_at", + "replication_method": "INCREMENTAL", + "key_properties": [ + "id" + ], + "schema": { + "properties": { + "search_name": { + "type": [ + "string", + "null" + ] + }, + "search_query": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "repo": { + "type": [ + "string", + "null" + ] + }, + "org": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "full_name": { + "type": [ + "string", + "null" + ] + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "owner": { + "properties": { + "login": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "avatar_url": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "site_admin": { + "type": [ + "boolean", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "license": { + "properties": { + "key": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "spdx_id": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "master_branch": { + "type": [ + "string", + "null" + ] + }, + "default_branch": { + "type": [ + "string", + "null" + ] + }, + "updated_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "created_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "pushed_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "git_url": { + "type": [ + "string", + "null" + ] + }, + "ssh_url": { + "type": [ + "string", + "null" + ] + }, + "clone_url": { + "type": [ + "string", + "null" + ] + }, + "homepage": { + "type": [ + "string", + "null" + ] + }, + "private": { + "type": [ + "boolean", + "null" + ] + }, + "archived": { + "type": [ + "boolean", + "null" + ] + }, + "disabled": { + "type": [ + "boolean", + "null" + ] + }, + "size": { + "type": [ + "integer", + "null" + ] + }, + "stargazers_count": { + "type": [ + "integer", + "null" + ] + }, + "fork": { + "type": [ + "boolean", + "null" + ] + }, + "topics": { + "items": { + "type": [ + "string" + ] + }, + "type": [ + "array", + "null" + ] + }, + "visibility": { + "type": [ + "string", + "null" + ] + }, + "language": { + "type": [ + "string", + "null" + ] + }, + "forks": { + "type": [ + "integer", + "null" + ] + }, + "forks_count": { + "type": [ + "integer", + "null" + ] + }, + "watchers": { + "type": [ + "integer", + "null" + ] + }, + "watchers_count": { + "type": [ + "integer", + "null" + ] + }, + "open_issues": { + "type": [ + "integer", + "null" + ] + }, + "network_count": { + "type": [ + "integer", + "null" + ] + }, + "subscribers_count": { + "type": [ + "integer", + "null" + ] + }, + "open_issues_count": { + "type": [ + "integer", + "null" + ] + }, + "allow_squash_merge": { + "type": [ + "boolean", + "null" + ] + }, + "allow_merge_commit": { + "type": [ + "boolean", + "null" + ] + }, + "allow_rebase_merge": { + "type": [ + "boolean", + "null" + ] + }, + "allow_auto_merge": { + "type": [ + "boolean", + "null" + ] + }, + "delete_branch_on_merge": { + "type": [ + "boolean", + "null" + ] + }, + "organization": { + "properties": { + "login": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "avatar_url": { + "type": [ + "string", + "null" + ] + }, + "gravatar_id": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "site_admin": { + "type": [ + "boolean", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + } + }, + "type": "object" + }, + "stream": "repositories", + "metadata": [ + { + "breadcrumb": [ + "properties", + "search_name" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "search_query" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "id" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "node_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "repo" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "org" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "name" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "full_name" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "description" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "html_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "owner" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "license" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "master_branch" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "default_branch" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "updated_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "created_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "pushed_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "git_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "ssh_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "clone_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "homepage" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "private" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "archived" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "disabled" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "size" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "stargazers_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "fork" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "topics" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "visibility" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "language" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "forks" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "forks_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "watchers" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "watchers_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "open_issues" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "network_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "subscribers_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "open_issues_count" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_squash_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_merge_commit" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_rebase_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "allow_auto_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "delete_branch_on_merge" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "organization" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "selected": true, + "table-key-properties": [ + "id" + ], + "valid-replication-keys": [ + "updated_at" + ] + } + } + ] + }, + { + "tap_stream_id": "review_comments", + "replication_method": "FULL_TABLE", + "key_properties": [ + "id" + ], + "schema": { + "properties": { + "org": { + "type": [ + "string", + "null" + ] + }, + "repo": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "pull_request_review_id": { + "type": [ + "integer", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "diff_hunk": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": [ + "string", + "null" + ] + }, + "position": { + "type": [ + "integer", + "null" + ] + }, + "original_position": { + "type": [ + "integer", + "null" + ] + }, + "commit_id": { + "type": [ + "string", + "null" + ] + }, + "original_commit_id": { + "type": [ + "string", + "null" + ] + }, + "in_reply_to_id": { + "type": [ + "integer", + "null" + ] + }, + "user": { + "properties": { + "login": { + "type": [ + "string", + "null" + ] + }, + "id": { + "type": [ + "integer", + "null" + ] + }, + "node_id": { + "type": [ + "string", + "null" + ] + }, + "avatar_url": { + "type": [ + "string", + "null" + ] + }, + "gravatar_id": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "site_admin": { + "type": [ + "boolean", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "body": { + "type": [ + "string", + "null" + ] + }, + "created_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "updated_at": { + "format": "date-time", + "type": [ + "string", + "null" + ] + }, + "html_url": { + "type": [ + "string", + "null" + ] + }, + "pull_request_url": { + "type": [ + "string", + "null" + ] + }, + "author_association": { + "type": [ + "string", + "null" + ] + }, + "_links": { + "properties": { + "self": { + "properties": { + "href": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "html": { + "properties": { + "href": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "pull_request": { + "properties": { + "href": { + "type": [ + "string", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + } + }, + "type": [ + "object", + "null" + ] + }, + "start_line": { + "type": [ + "integer", + "null" + ] + }, + "original_start_line": { + "type": [ + "integer", + "null" + ] + }, + "start_side": { + "type": [ + "string", + "null" + ] + }, + "line": { + "type": [ + "integer", + "null" + ] + }, + "original_line": { + "type": [ + "integer", + "null" + ] + }, + "side": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "stream": "review_comments", + "metadata": [ + { + "breadcrumb": [ + "properties", + "org" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "repo" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "pull_request_review_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "id" + ], + "metadata": { + "inclusion": "automatic" + } + }, + { + "breadcrumb": [ + "properties", + "node_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "diff_hunk" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "path" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "position" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_position" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "commit_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_commit_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "in_reply_to_id" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "user" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "body" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "created_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "updated_at" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "html_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "pull_request_url" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "author_association" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "_links" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "start_line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_start_line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "start_side" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "original_line" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [ + "properties", + "side" + ], + "metadata": { + "inclusion": "available" + } + }, + { + "breadcrumb": [], + "metadata": { + "inclusion": "available", + "selected": true, + "table-key-properties": [ + "id" + ] + } + } + ] + } + ] +} diff --git a/.circleci/integration/tap-github/config-template.json b/.circleci/integration/tap-github/config-template.json index 5c7a908..d9d9b62 100644 --- a/.circleci/integration/tap-github/config-template.json +++ b/.circleci/integration/tap-github/config-template.json @@ -1,4 +1,4 @@ { - "access_token": "REPLACE_ME", - "repository": "datamill-co/target-postgres" -} \ No newline at end of file + "start_date": "2022-04-12", + "repositories": ["datamill-co/target-postgres"] +} diff --git a/.circleci/integration/tap-github/properties.json b/.circleci/integration/tap-github/properties.json deleted file mode 100644 index 6f2812c..0000000 --- a/.circleci/integration/tap-github/properties.json +++ /dev/null @@ -1,2414 +0,0 @@ -{ - "streams": [ - { - "stream": "stargazers", - "tap_stream_id": "stargazers", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "starred_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "user_id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "user_id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "starred_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user_id" - ], - "metadata": { - "inclusion": "automatic" - } - } - ], - "key_properties": [ - "user_id" - ] - }, - { - "stream": "assignees", - "tap_stream_id": "assignees", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "_sdc_repository": { - "type": [ - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "login" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "type" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "reviews", - "tap_stream_id": "reviews", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "commit_id": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "pull_request_url": { - "type": [ - "null", - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "state" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commit_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request_url" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "issues", - "tap_stream_id": "issues", - "schema": { - "selected": true, - "properties": { - "state": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "labels": { - "type": [ - "null", - "array" - ], - "items": { - "type": "object", - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "description": { - "type": [ - "null", - "string" - ] - }, - "color": { - "type": [ - "null", - "string" - ] - }, - "default": { - "type": [ - "null", - "boolean" - ] - } - } - } - }, - "repository_url": { - "type": [ - "null", - "string" - ] - }, - "number": { - "type": [ - "null", - "integer" - ] - }, - "labels_url": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "assignee": { - "type": [ - "null", - "object" - ], - "properties": {} - }, - "updated_at": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "author_association": { - "type": [ - "null", - "string" - ] - }, - "locked": { - "type": [ - "null", - "boolean" - ] - }, - "events_url": { - "type": [ - "null", - "string" - ] - }, - "pull_request": { - "properties": { - "diff_url": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "patch_url": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "comments": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ] - }, - "_sdc_repository": { - "type": [ - "string" - ] - }, - "user": { - "properties": { - "repos_url": { - "type": [ - "null", - "string" - ] - }, - "starred_url": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "received_events_url": { - "type": [ - "null", - "string" - ] - }, - "site_admin": { - "type": [ - "null", - "boolean" - ] - }, - "gravatar_id": { - "type": [ - "null", - "string" - ] - }, - "following_url": { - "type": [ - "null", - "string" - ] - }, - "avatar_url": { - "type": [ - "null", - "string" - ] - }, - "events_url": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "login": { - "type": [ - "null", - "string" - ] - }, - "organizations_url": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "subscriptions_url": { - "type": [ - "null", - "string" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "followers_url": { - "type": [ - "null", - "string" - ] - }, - "gists_url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "comments_url": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "state" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "labels" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "repository_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "number" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "closed_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "labels_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "title" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "assignee" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "author_association" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "locked" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "events_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "node_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "comments" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "milestone" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "review_comments", - "tap_stream_id": "review_comments", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "pull_request_review_id": { - "type": [ - "null", - "integer" - ] - }, - "diff_hunk": { - "type": [ - "null", - "string" - ] - }, - "path": { - "type": [ - "null", - "string" - ] - }, - "position": { - "type": [ - "null", - "integer" - ] - }, - "original_position": { - "type": [ - "null", - "integer" - ] - }, - "commit_id": { - "type": [ - "null", - "string" - ] - }, - "original_commit_id": { - "type": [ - "null", - "string" - ] - }, - "in_reply_to_id": { - "type": [ - "null", - "integer" - ] - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "pull_request_url": { - "type": [ - "null", - "string" - ] - }, - "assignee": { - "type": [ - "null", - "string" - ] - }, - "assignees": { - "type": [ - "null", - "string" - ] - }, - "author_association": { - "type": [ - "null", - "string" - ] - }, - "base": { - "type": [ - "null", - "string" - ] - }, - "comments_url": { - "type": [ - "null", - "string" - ] - }, - "commits_url": { - "type": [ - "null", - "string" - ] - }, - "diff_url": { - "type": [ - "null", - "string" - ] - }, - "head": { - "type": [ - "null", - "string" - ] - }, - "issue_url": { - "type": [ - "null", - "string" - ] - }, - "labels": { - "type": [ - "null", - "string" - ] - }, - "locked": { - "type": [ - "null", - "string" - ] - }, - "merge_commit_sha": { - "type": [ - "null", - "string" - ] - }, - "milestone": { - "type": [ - "null", - "string" - ] - }, - "patch_url": { - "type": [ - "null", - "string" - ] - }, - "requested_reviewers": { - "type": [ - "null", - "string" - ] - }, - "requested_teams": { - "type": [ - "null", - "string" - ] - }, - "review_comment_url": { - "type": [ - "null", - "string" - ] - }, - "review_comments_url": { - "type": [ - "null", - "string" - ] - }, - "statuses_url": { - "type": [ - "null", - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "node_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request_review_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "diff_hunk" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "path" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "position" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "original_position" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commit_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "original_commit_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "in_reply_to_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "pull_request_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "assignee" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "assignees" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "author_association" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "base" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commits_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "diff_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "head" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "issue_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "labels" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "locked" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "merge_commit_sha" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "milestone" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "patch_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "requested_reviewers" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "requested_teams" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "review_comment_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "review_comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "statuses_url" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "collaborators", - "tap_stream_id": "collaborators", - "schema": { - "selected": false, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "_sdc_repository": { - "type": [ - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "login" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "type" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "comments", - "tap_stream_id": "comments", - "schema": { - "selected": true, - "type": "object", - "properties": { - "id": { - "type": [ - "null", - "integer" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "home_url": { - "type": [ - "null", - "string" - ] - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "issue_url": { - "type": [ - "null", - "string" - ] - }, - "author_association": { - "type": [ - "null", - "string" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "node_id": { - "type": [ - "null", - "string" - ] - }, - "avatar_url": { - "type": [ - "null", - "string" - ] - }, - "gravatar_id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "html_url": { - "type": [ - "null", - "string" - ] - }, - "followers_url": { - "type": [ - "null", - "string" - ] - }, - "following_url": { - "type": [ - "null", - "string" - ] - }, - "gists_url": { - "type": [ - "null", - "string" - ] - }, - "starred_url": { - "type": [ - "null", - "string" - ] - }, - "subscriptions_url": { - "type": [ - "null", - "string" - ] - }, - "organizations_url": { - "type": [ - "null", - "string" - ] - }, - "repos_url": { - "type": [ - "null", - "string" - ] - }, - "events_url": { - "type": [ - "null", - "string" - ] - }, - "received_events_url": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "site_admin": { - "type": [ - "null", - "string" - ] - } - } - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "_sdc_repository": { - "type": [ - "string" - ] - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "node_id" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "home_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "issue_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "author_association" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "pull_requests", - "tap_stream_id": "pull_requests", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "number": { - "type": [ - "null", - "integer" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "body": { - "type": [ - "null", - "string" - ] - }, - "user": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "login": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - } - } - }, - "merged_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "closed_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "created_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updated_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "id" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "id" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "number" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "state" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "title" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "body" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "user" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "merged_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "closed_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "created_at" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "updated_at" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "id" - ] - }, - { - "stream": "commits", - "tap_stream_id": "commits", - "schema": { - "selected": true, - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "_sdc_repository": { - "type": [ - "string" - ] - }, - "sha": { - "type": [ - "null", - "string" - ], - "description": "The git commit hash" - }, - "url": { - "type": [ - "null", - "string" - ] - }, - "parents": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "sha": { - "type": [ - "null", - "string" - ], - "description": "The git hash of the parent commit" - }, - "url": { - "type": [ - "null", - "string" - ], - "description": "The URL to the parent commit" - }, - "html_url": { - "type": [ - "null", - "string" - ], - "description": "The HTML URL to the parent commit" - } - } - } - }, - "html_url": { - "type": [ - "null", - "string" - ], - "description": "The HTML URL to the commit" - }, - "comments_url": { - "type": [ - "null", - "string" - ], - "description": "The URL to the commit's comments page" - }, - "commit": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "url": { - "type": [ - "null", - "string" - ], - "description": "The URL to the commit" - }, - "tree": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "sha": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": [ - "null", - "string" - ] - } - } - }, - "author": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "date": { - "type": [ - "null", - "string" - ], - "format": "date-time", - "description": "The date the author committed the change" - }, - "name": { - "type": [ - "null", - "string" - ], - "description": "The author's name" - }, - "email": { - "type": [ - "null", - "string" - ], - "description": "The author's email" - } - } - }, - "message": { - "type": [ - "null", - "string" - ], - "description": "The commit message" - }, - "committer": { - "type": [ - "null", - "object" - ], - "additionalProperties": false, - "properties": { - "date": { - "type": [ - "null", - "string" - ], - "format": "date-time", - "description": "The date the committer committed the change" - }, - "name": { - "type": [ - "null", - "string" - ], - "description": "The committer's name" - }, - "email": { - "type": [ - "null", - "string" - ], - "description": "The committer's email" - } - } - }, - "comment_count": { - "type": [ - "null", - "integer" - ], - "description": "The number of comments on the commit" - } - } - } - } - }, - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "sha" - ] - } - }, - { - "breadcrumb": [ - "properties", - "_sdc_repository" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "sha" - ], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [ - "properties", - "url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "parents" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "html_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "comments_url" - ], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [ - "properties", - "commit" - ], - "metadata": { - "inclusion": "available" - } - } - ], - "key_properties": [ - "sha" - ] - } - ] -} From e0004e5fe758844378dcd50e0227a3dae2e83a89 Mon Sep 17 00:00:00 2001 From: Damian Karzon Date: Sun, 17 Apr 2022 08:25:55 +1000 Subject: [PATCH 5/5] Add SQL files for before/after hooks (in addition to SQL strings) (#200) Co-authored-by: Eric Boucher --- .circleci/config.yml | 29 ++++++++++++++++++++++------- README.md | 2 ++ target_postgres/target_tools.py | 7 +++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35fa136..2449387 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,10 +82,25 @@ restore__cache: &restore__cache keys: - *cache +# Simple checkout command to pull external forks. +# The CircleCI util does not work without setting up SSH keys +# which we technically do not need for open-source repos. +checkout_command: &checkout_command + run: + name: checkout + command: | + git clone https://github.com/datamill-co/target-postgres . + if [[ "$CIRCLE_BRANCH" =~ ^pull\/* ]]; then + git fetch origin refs/pull/${CIRCLE_PR_NUMBER}/head + git checkout ${CIRCLE_SHA1} + else + git checkout ${CIRCLE_BRANCH} + fi + test__base: &test__base working_directory: /code/ steps: - - run: git clone https://github.com/datamill-co/target-postgres . + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -111,7 +126,7 @@ jobs: docker: - image: *py steps: - - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} + - *checkout_command - *restore__cache - run: @@ -226,7 +241,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres steps: - - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -318,7 +333,7 @@ jobs: POSTGRES_DB: target_postgres_test POSTGRES_PASSWORD: postgres steps: - - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -343,7 +358,7 @@ jobs: docker: - image: *py steps: - - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -368,7 +383,7 @@ jobs: docker: - image: *py steps: - - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} + - *checkout_command - *restore__cache - attach_workspace: at: "./" @@ -398,7 +413,7 @@ jobs: docker: - image: *py steps: - - run: git clone https://github.com/datamill-co/target-postgres /code/ && git checkout ${CIRCLE_BRANCH} + - *checkout_command - *restore__cache - attach_workspace: at: "./" diff --git a/README.md b/README.md index 6e34418..cd2baf6 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ here. | `add_upsert_indexes` | `["boolean", "null"]` | `True` | Whether the Target should create column indexes on the important columns used during data loading. These indexes will make data loading slightly slower but the deduplication phase much faster. Defaults to on for better baseline performance. | | `before_run_sql` | `["string", "null"]` | `None` | Raw SQL statement(s) to execute as soon as the connection to Postgres is opened by the target. Useful for setup like `SET ROLE` or other connection state that is important. | | `after_run_sql` | `["string", "null"]` | `None` | Raw SQL statement(s) to execute as soon as the connection to Postgres is opened by the target. Useful for setup like `SET ROLE` or other connection state that is important. | +| `before_run_sql_file` | `["string", "null"]` | `None` | Similar to `before_run_sql` but reads an external file instead of SQL in the JSON config file. | +| `after_run_sql_file` | `["string", "null"]` | `None` | Similar to `after_run_sql` but reads an external file instead of SQL in the JSON config file. | ### Supported Versions diff --git a/target_postgres/target_tools.py b/target_postgres/target_tools.py index c01cc40..3cc6070 100644 --- a/target_postgres/target_tools.py +++ b/target_postgres/target_tools.py @@ -188,3 +188,10 @@ def _run_sql_hook(hook_name, config, target): with target.conn.cursor() as cur: cur.execute(config[hook_name]) LOGGER.debug('{} SQL executed'.format(hook_name)) + + hook_file = hook_name + '_file' + if hook_file in config: + with open(config[hook_file]) as f: + with target.conn.cursor() as cur: + cur.execute(f.read()) + LOGGER.debug('{} SQL file executed'.format(hook_file))