From 86af60ad87d537b17e4ce6ec7a5eac0d637fb32d Mon Sep 17 00:00:00 2001 From: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> Date: Tue, 25 Jun 2024 21:51:39 +0200 Subject: [PATCH 01/33] fix: Added missing type (#4315) Added missing type Signed-off-by: Daniele Martinoli <86618610+dmartinol@users.noreply.github.com> --- sdk/python/feast/diff/registry_diff.py | 2 ++ sdk/python/feast/feast_object.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/diff/registry_diff.py b/sdk/python/feast/diff/registry_diff.py index b608757496..9236b087d4 100644 --- a/sdk/python/feast/diff/registry_diff.py +++ b/sdk/python/feast/diff/registry_diff.py @@ -20,6 +20,7 @@ OnDemandFeatureView as OnDemandFeatureViewProto, ) from feast.protos.feast.core.OnDemandFeatureView_pb2 import OnDemandFeatureViewSpec +from feast.protos.feast.core.SavedDataset_pb2 import SavedDataset as SavedDatasetProto from feast.protos.feast.core.StreamFeatureView_pb2 import ( StreamFeatureView as StreamFeatureViewProto, ) @@ -109,6 +110,7 @@ def tag_objects_for_keep_delete_update_add( OnDemandFeatureViewProto, StreamFeatureViewProto, ValidationReferenceProto, + SavedDatasetProto, ) diff --git a/sdk/python/feast/feast_object.py b/sdk/python/feast/feast_object.py index 2d06d8d669..d9505dcb9f 100644 --- a/sdk/python/feast/feast_object.py +++ b/sdk/python/feast/feast_object.py @@ -11,11 +11,12 @@ from .protos.feast.core.FeatureService_pb2 import FeatureServiceSpec from .protos.feast.core.FeatureView_pb2 import FeatureViewSpec from .protos.feast.core.OnDemandFeatureView_pb2 import OnDemandFeatureViewSpec +from .protos.feast.core.SavedDataset_pb2 import SavedDatasetSpec from .protos.feast.core.StreamFeatureView_pb2 import StreamFeatureViewSpec from .protos.feast.core.ValidationProfile_pb2 import ( ValidationReference as ValidationReferenceProto, ) -from .saved_dataset import ValidationReference +from .saved_dataset import SavedDataset, ValidationReference from .stream_feature_view import StreamFeatureView # Convenience type representing all Feast objects @@ -28,6 +29,7 @@ FeatureService, DataSource, ValidationReference, + SavedDataset, ] FeastObjectSpecProto = Union[ @@ -38,4 +40,5 @@ FeatureServiceSpec, DataSourceProto, ValidationReferenceProto, + SavedDatasetSpec, ] From 372fd75394c36ab4d864758eedcfa94aafdb6a2e Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Thu, 27 Jun 2024 02:27:13 +0400 Subject: [PATCH 02/33] chore: Remove existing providers from the repo (#4298) remove providers Signed-off-by: tokoko --- sdk/python/feast/errors.py | 12 ---- sdk/python/feast/infra/aws.py | 9 --- .../feast/infra/contrib/azure_provider.py | 72 ------------------- sdk/python/feast/infra/gcp.py | 9 --- sdk/python/feast/infra/local.py | 23 ------ .../feast/infra/passthrough_provider.py | 13 ++++ sdk/python/feast/infra/provider.py | 8 +-- sdk/python/feast/repo_config.py | 66 ++--------------- .../offline_stores/test_offline_store.py | 4 +- .../infra/scaffolding/test_repo_config.py | 59 ++++----------- 10 files changed, 40 insertions(+), 235 deletions(-) delete mode 100644 sdk/python/feast/infra/aws.py delete mode 100644 sdk/python/feast/infra/contrib/azure_provider.py delete mode 100644 sdk/python/feast/infra/gcp.py delete mode 100644 sdk/python/feast/infra/local.py diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 22de402f20..6083b3d554 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -119,23 +119,11 @@ def __init__(self, provider_name): super().__init__(f"Provider '{provider_name}' is not implemented") -class FeastProviderNotSetError(Exception): - def __init__(self): - super().__init__("Provider is not set, but is required") - - class FeastRegistryNotSetError(Exception): def __init__(self): super().__init__("Registry is not set, but is required") -class FeastFeatureServerTypeSetError(Exception): - def __init__(self, feature_server_type: str): - super().__init__( - f"Feature server type was set to {feature_server_type}, but the type should be determined by the provider" - ) - - class FeastFeatureServerTypeInvalidError(Exception): def __init__(self, feature_server_type: str): super().__init__( diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py deleted file mode 100644 index 47fec9b05b..0000000000 --- a/sdk/python/feast/infra/aws.py +++ /dev/null @@ -1,9 +0,0 @@ -from feast.infra.passthrough_provider import PassthroughProvider - - -class AwsProvider(PassthroughProvider): - """ - This class only exists for backwards compatibility. - """ - - pass diff --git a/sdk/python/feast/infra/contrib/azure_provider.py b/sdk/python/feast/infra/contrib/azure_provider.py deleted file mode 100644 index ac56a2b33e..0000000000 --- a/sdk/python/feast/infra/contrib/azure_provider.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. -from datetime import datetime -from typing import Callable - -from tqdm import tqdm - -from feast.feature_view import FeatureView -from feast.infra.passthrough_provider import PassthroughProvider -from feast.infra.registry.base_registry import BaseRegistry -from feast.repo_config import RepoConfig -from feast.utils import ( - _convert_arrow_to_proto, - _get_column_names, - _run_pyarrow_field_mapping, -) - -DEFAULT_BATCH_SIZE = 10_000 - - -class AzureProvider(PassthroughProvider): - def materialize_single_feature_view( - self, - config: RepoConfig, - feature_view: FeatureView, - start_date: datetime, - end_date: datetime, - registry: BaseRegistry, - project: str, - tqdm_builder: Callable[[int], tqdm], - ) -> None: - # TODO(kevjumba): untested - entities = [] - for entity_name in feature_view.entities: - entities.append(registry.get_entity(entity_name, project)) - - ( - join_key_columns, - feature_name_columns, - event_timestamp_column, - created_timestamp_column, - ) = _get_column_names(feature_view, entities) - - offline_job = self.offline_store.pull_latest_from_table_or_query( - config=config, - data_source=feature_view.batch_source, - join_key_columns=join_key_columns, - feature_name_columns=feature_name_columns, - timestamp_field=event_timestamp_column, - created_timestamp_column=created_timestamp_column, - start_date=start_date, - end_date=end_date, - ) - - table = offline_job.to_arrow() - - if feature_view.batch_source.field_mapping is not None: - table = _run_pyarrow_field_mapping( - table, feature_view.batch_source.field_mapping - ) - - join_keys = {entity.join_key: entity.value_type for entity in entities} - - with tqdm_builder(table.num_rows) as pbar: - for batch in table.to_batches(DEFAULT_BATCH_SIZE): - rows_to_write = _convert_arrow_to_proto(batch, feature_view, join_keys) - self.online_write_batch( - self.repo_config, - feature_view, - rows_to_write, - lambda x: pbar.update(x), - ) diff --git a/sdk/python/feast/infra/gcp.py b/sdk/python/feast/infra/gcp.py deleted file mode 100644 index 512378237a..0000000000 --- a/sdk/python/feast/infra/gcp.py +++ /dev/null @@ -1,9 +0,0 @@ -from feast.infra.passthrough_provider import PassthroughProvider - - -class GcpProvider(PassthroughProvider): - """ - This class only exists for backwards compatibility. - """ - - pass diff --git a/sdk/python/feast/infra/local.py b/sdk/python/feast/infra/local.py deleted file mode 100644 index 1226ceaf37..0000000000 --- a/sdk/python/feast/infra/local.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import List - -from feast.infra.infra_object import Infra, InfraObject -from feast.infra.passthrough_provider import PassthroughProvider -from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto -from feast.repo_config import RepoConfig - - -class LocalProvider(PassthroughProvider): - """ - This class only exists for backwards compatibility. - """ - - def plan_infra( - self, config: RepoConfig, desired_registry_proto: RegistryProto - ) -> Infra: - infra = Infra() - if self.online_store: - infra_objects: List[InfraObject] = self.online_store.plan( - config, desired_registry_proto - ) - infra.infra_objects += infra_objects - return infra diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index e707f9495d..bad6f86cc6 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -12,6 +12,7 @@ from feast.feature_logging import FeatureServiceLoggingSource from feast.feature_service import FeatureService from feast.feature_view import FeatureView +from feast.infra.infra_object import Infra, InfraObject from feast.infra.materialization.batch_materialization_engine import ( BatchMaterializationEngine, MaterializationJobStatus, @@ -22,6 +23,7 @@ from feast.infra.online_stores.helpers import get_online_store_from_config from feast.infra.provider import Provider from feast.infra.registry.base_registry import BaseRegistry +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import BATCH_ENGINE_CLASS_FOR_TYPE, RepoConfig @@ -103,6 +105,17 @@ def batch_engine(self) -> BatchMaterializationEngine: self._batch_engine = _batch_engine return _batch_engine + def plan_infra( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> Infra: + infra = Infra() + if self.online_store: + infra_objects: List[InfraObject] = self.online_store.plan( + config, desired_registry_proto + ) + infra.infra_objects += infra_objects + return infra + def update_infra( self, project: str, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 93077f40b9..75afd6bba8 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -22,10 +22,10 @@ from feast.saved_dataset import SavedDataset PROVIDERS_CLASS_FOR_TYPE = { - "gcp": "feast.infra.gcp.GcpProvider", - "aws": "feast.infra.aws.AwsProvider", - "local": "feast.infra.local.LocalProvider", - "azure": "feast.infra.contrib.azure_provider.AzureProvider", + "gcp": "feast.infra.passthrough_provider.PassthroughProvider", + "aws": "feast.infra.passthrough_provider.PassthroughProvider", + "local": "feast.infra.passthrough_provider.PassthroughProvider", + "azure": "feast.infra.passthrough_provider.PassthroughProvider", } diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 1c8041b4dd..99b90a09a5 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -18,10 +18,8 @@ from feast.errors import ( FeastFeatureServerTypeInvalidError, - FeastFeatureServerTypeSetError, FeastOfflineStoreInvalidName, FeastOnlineStoreInvalidName, - FeastProviderNotSetError, FeastRegistryNotSetError, FeastRegistryTypeInvalidError, ) @@ -85,10 +83,6 @@ "local": "feast.infra.feature_servers.local_process.config.LocalFeatureServerConfig", } -FEATURE_SERVER_TYPE_FOR_PROVIDER = { - "local": "local", -} - class FeastBaseModel(BaseModel): """Feast Pydantic Configuration Class""" @@ -138,7 +132,7 @@ class RepoConfig(FeastBaseModel): provider account, as long as they have different project ids. """ - provider: StrictStr + provider: StrictStr = "local" """ str: local or gcp or aws """ registry_config: Any = Field(alias="registry", default="data/registry.db") @@ -191,30 +185,10 @@ def __init__(self, **data: Any): self.registry_config = data["registry"] self._offline_store = None - if "offline_store" in data: - self.offline_config = data["offline_store"] - else: - if data["provider"] == "local": - self.offline_config = "file" - elif data["provider"] == "gcp": - self.offline_config = "bigquery" - elif data["provider"] == "aws": - self.offline_config = "redshift" - elif data["provider"] == "azure": - self.offline_config = "mssql" + self.offline_config = data.get("offline_store", "file") self._online_store = None - if "online_store" in data: - self.online_config = data["online_store"] - else: - if data["provider"] == "local": - self.online_config = "sqlite" - elif data["provider"] == "gcp": - self.online_config = "datastore" - elif data["provider"] == "aws": - self.online_config = "dynamodb" - elif data["provider"] == "rockset": - self.online_config = "rockset" + self.online_config = data.get("online_store", "sqlite") self._batch_engine = None if "batch_engine" in data: @@ -325,20 +299,11 @@ def _validate_online_store_config(cls, values: Any) -> Any: values["online_store"] = None return values - # Make sure that the provider configuration is set. We need it to set the defaults - if "provider" not in values: - raise FeastProviderNotSetError() - # Set the default type # This is only direct reference to a provider or online store that we should have # for backwards compatibility. if "type" not in values["online_store"]: - if values["provider"] == "local": - values["online_store"]["type"] = "sqlite" - elif values["provider"] == "gcp": - values["online_store"]["type"] = "datastore" - elif values["provider"] == "aws": - values["online_store"]["type"] = "dynamodb" + values["online_store"]["type"] = "sqlite" online_store_type = values["online_store"]["type"] @@ -361,20 +326,9 @@ def _validate_offline_store_config(cls, values: Any) -> Any: if not isinstance(values["offline_store"], Dict): return values - # Make sure that the provider configuration is set. We need it to set the defaults - if "provider" not in values: - raise FeastProviderNotSetError() - # Set the default type if "type" not in values["offline_store"]: - if values["provider"] == "local": - values["offline_store"]["type"] = "file" - elif values["provider"] == "gcp": - values["offline_store"]["type"] = "bigquery" - elif values["provider"] == "aws": - values["offline_store"]["type"] = "redshift" - if values["provider"] == "azure": - values["offline_store"]["type"] = "mssql" + values["offline_store"]["type"] = "file" offline_store_type = values["offline_store"]["type"] @@ -398,15 +352,7 @@ def _validate_feature_server_config(cls, values: Any) -> Any: if not isinstance(values["feature_server"], Dict): return values - # Make sure that the provider configuration is set. We need it to set the defaults - if "provider" not in values: - raise FeastProviderNotSetError() - - default_type = FEATURE_SERVER_TYPE_FOR_PROVIDER.get(values["provider"]) - defined_type = values["feature_server"].get("type", default_type) - # Make sure that the type is either not set, or set correctly, since it's defined by the provider - if defined_type not in (default_type, "local"): - raise FeastFeatureServerTypeSetError(defined_type) + defined_type = values["feature_server"].get("type", "local") values["feature_server"]["type"] = defined_type # Validate the dict to ensure one of the union types match diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index fd50d37632..3589c8a3fa 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -124,7 +124,7 @@ def retrieval_job(request, environment): iam_role="arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631", workgroup="", ) - config = environment.config.copy( + config = environment.config.model_copy( update={"offline_config": offline_store_config} ) return RedshiftRetrievalJob( @@ -147,7 +147,7 @@ def retrieval_job(request, environment): storage_integration_name="FEAST_S3", blob_export_location="s3://feast-snowflake-offload/export", ) - config = environment.config.copy( + config = environment.config.model_copy( update={"offline_config": offline_store_config} ) environment.project = "project" diff --git a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py index e1839fbd8b..98d82ce357 100644 --- a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py +++ b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py @@ -33,36 +33,6 @@ def _test_config(config_text, expect_error: Optional[str]): return rc -def test_nullable_online_store_aws(): - _test_config( - dedent( - """ - project: foo - registry: "registry.db" - provider: aws - online_store: null - entity_key_serialization_version: 2 - """ - ), - expect_error="4 validation errors for RepoConfig\nregion\n Field required", - ) - - -def test_nullable_online_store_gcp(): - _test_config( - dedent( - """ - project: foo - registry: "registry.db" - provider: gcp - online_store: null - entity_key_serialization_version: 2 - """ - ), - expect_error=None, - ) - - def test_nullable_online_store_local(): _test_config( dedent( @@ -125,20 +95,6 @@ def test_local_config_with_full_online_class_directly(): assert isinstance(c.online_store, SqliteOnlineStoreConfig) -def test_gcp_config(): - _test_config( - dedent( - """ - project: foo - registry: gs://registry.db - provider: gcp - entity_key_serialization_version: 2 - """ - ), - expect_error=None, - ) - - def test_extra_field(): _test_config( dedent( @@ -224,3 +180,18 @@ def test_invalid_project_name(): ), expect_error="alphanumerical values ", ) + + +def test_no_provider(): + _test_config( + dedent( + """ + project: foo + registry: "registry.db" + online_store: + path: "blah" + entity_key_serialization_version: 2 + """ + ), + expect_error=None, + ) From 2c3894693e9079b8ad7873b139b30440c919e913 Mon Sep 17 00:00:00 2001 From: okramarenko <97118627+okramarenko@users.noreply.github.com> Date: Thu, 27 Jun 2024 01:27:33 +0300 Subject: [PATCH 03/33] feat: Add SingleStore as an OnlineStore (#4285) Add SingleStore as an OnlineStore Signed-off-by: Olha Kramarenko --- Makefile | 11 + docs/SUMMARY.md | 1 + docs/reference/online-stores/README.md | 3 + docs/reference/online-stores/singlestore.md | 51 ++++ .../feast.infra.online_stores.contrib.rst | 8 + .../feast.infra.registry.contrib.postgres.rst | 21 ++ .../source/feast.infra.registry.contrib.rst | 1 + .../singlestore_online_store/singlestore.py | 235 ++++++++++++++++++ .../contrib/singlestore_repo_configuration.py | 10 + sdk/python/feast/repo_config.py | 1 + .../requirements/py3.10-ci-requirements.txt | 17 +- .../requirements/py3.11-ci-requirements.txt | 16 +- .../requirements/py3.9-ci-requirements.txt | 17 +- .../universal/online_store/singlestore.py | 43 ++++ setup.py | 4 + 15 files changed, 433 insertions(+), 6 deletions(-) create mode 100644 docs/reference/online-stores/singlestore.md create mode 100644 sdk/python/docs/source/feast.infra.registry.contrib.postgres.rst create mode 100644 sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/singlestore_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/singlestore.py diff --git a/Makefile b/Makefile index 2ad693c7a1..39406cc17d 100644 --- a/Makefile +++ b/Makefile @@ -331,6 +331,17 @@ test-python-universal-cassandra-no-cloud-providers: not test_snowflake" \ sdk/python/tests +test-python-universal-singlestore-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.singlestore_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.singlestore \ + python -m pytest -n 8 --integration \ + -k "not test_universal_cli and \ + not gcs_registry and \ + not s3_registry and \ + not test_snowflake" \ + sdk/python/tests + test-python-universal: python -m pytest -n 8 --integration sdk/python/tests diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 3f0506cf1e..a40c60d97c 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -103,6 +103,7 @@ * [Rockset (contrib)](reference/online-stores/rockset.md) * [Hazelcast (contrib)](reference/online-stores/hazelcast.md) * [ScyllaDB (contrib)](reference/online-stores/scylladb.md) + * [SingleStore (contrib)](reference/online-stores/singlestore.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index b5f4eb8de8..0acf6701f9 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -64,4 +64,7 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli {% content-ref url="remote.md" %} [remote.md](remote.md) + +{% content-ref url="singlestore.md" %} +[singlestore.md](singlestore.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/singlestore.md b/docs/reference/online-stores/singlestore.md new file mode 100644 index 0000000000..1777787f22 --- /dev/null +++ b/docs/reference/online-stores/singlestore.md @@ -0,0 +1,51 @@ +# SingleStore online store (contrib) + +## Description + +The SingleStore online store provides support for materializing feature values into a SingleStore database for serving online features. + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[singlestore]'`. You can get started by then running `feast init` and then setting the `feature_store.yaml` as described below. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: singlestore + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD +``` +{% endcode %} + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the SingleStore online store. + +| | SingleStore | +| :-------------------------------------------------------- | :----------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index d614438e3d..9d301fcd0d 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -89,6 +89,14 @@ feast.infra.online\_stores.contrib.postgres\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.singlestore\_repo\_configuration module +-------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.singlestore_repo_configuration + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/sdk/python/docs/source/feast.infra.registry.contrib.postgres.rst b/sdk/python/docs/source/feast.infra.registry.contrib.postgres.rst new file mode 100644 index 0000000000..3f31990805 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.registry.contrib.postgres.rst @@ -0,0 +1,21 @@ +feast.infra.registry.contrib.postgres package +============================================= + +Submodules +---------- + +feast.infra.registry.contrib.postgres.postgres\_registry\_store module +---------------------------------------------------------------------- + +.. automodule:: feast.infra.registry.contrib.postgres.postgres_registry_store + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.registry.contrib.postgres + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.registry.contrib.rst b/sdk/python/docs/source/feast.infra.registry.contrib.rst index 83417109b8..44b89736ad 100644 --- a/sdk/python/docs/source/feast.infra.registry.contrib.rst +++ b/sdk/python/docs/source/feast.infra.registry.contrib.rst @@ -8,6 +8,7 @@ Subpackages :maxdepth: 4 feast.infra.registry.contrib.azure + feast.infra.registry.contrib.postgres Module contents --------------- diff --git a/sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py b/sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py new file mode 100644 index 0000000000..e17a059c1a --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/singlestore_online_store/singlestore.py @@ -0,0 +1,235 @@ +from __future__ import absolute_import + +from collections import defaultdict +from datetime import datetime +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple + +import pytz +import singlestoredb +from pydantic import StrictStr +from singlestoredb.connection import Connection, Cursor +from singlestoredb.exceptions import InterfaceError + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel + + +class SingleStoreOnlineStoreConfig(FeastConfigBaseModel): + """ + Configuration for the SingleStore online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ + + type: Literal["singlestore"] = "singlestore" + + host: Optional[StrictStr] = None + user: Optional[StrictStr] = None + password: Optional[StrictStr] = None + database: Optional[StrictStr] = None + port: Optional[int] = None + + +class SingleStoreOnlineStore(OnlineStore): + """ + An online store implementation that uses SingleStore. + NOTE: The class *must* end with the `OnlineStore` suffix. + """ + + _conn: Optional[Connection] = None + + def _init_conn(self, config: RepoConfig) -> Connection: + online_store_config = config.online_store + assert isinstance(online_store_config, SingleStoreOnlineStoreConfig) + return singlestoredb.connect( + host=online_store_config.host or "127.0.0.1", + user=online_store_config.user or "test", + password=online_store_config.password or "test", + database=online_store_config.database or "feast", + port=online_store_config.port or 3306, + autocommit=True, + ) + + def _get_cursor(self, config: RepoConfig) -> Any: + # This will try to reconnect also. + # In case it fails, we will have to create a new connection. + if not self._conn: + self._conn = self._init_conn(config) + try: + self._conn.ping(reconnect=True) + except InterfaceError: + self._conn = self._init_conn(config) + return self._conn.cursor() + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + project = config.project + with self._get_cursor(config) as cur: + insert_values = [] + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=2, + ).hex() + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) + + for feature_name, val in values.items(): + insert_values.append( + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + timestamp, + created_ts, + ) + ) + # Control the batch so that we can update the progress + batch_size = 50000 + for i in range(0, len(insert_values), batch_size): + current_batch = insert_values[i : i + batch_size] + cur.executemany( + f""" + INSERT INTO {_table_id(project, table)} + (entity_key, feature_name, value, event_ts, created_ts) + values (%s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + value = VALUES(value), + event_ts = VALUES(event_ts), + created_ts = VALUES(created_ts); + """, + current_batch, + ) + if progress: + progress(len(current_batch)) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + project = config.project + result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + with self._get_cursor(config) as cur: + keys = [] + for entity_key in entity_keys: + keys.append( + serialize_entity_key( + entity_key, + entity_key_serialization_version=2, + ).hex() + ) + + if not requested_features: + entity_key_placeholders = ",".join(["%s" for _ in keys]) + cur.execute( + f""" + SELECT entity_key, feature_name, value, event_ts FROM {_table_id(project, table)} + WHERE entity_key IN ({entity_key_placeholders}) + ORDER BY event_ts; + """, + tuple(keys), + ) + else: + entity_key_placeholders = ",".join(["%s" for _ in keys]) + requested_features_placeholders = ",".join( + ["%s" for _ in requested_features] + ) + cur.execute( + f""" + SELECT entity_key, feature_name, value, event_ts FROM {_table_id(project, table)} + WHERE entity_key IN ({entity_key_placeholders}) and feature_name IN ({requested_features_placeholders}) + ORDER BY event_ts; + """, + tuple(keys + requested_features), + ) + rows = cur.fetchall() or [] + + # Since we don't know the order returned from MySQL we'll need + # to construct a dict to be able to quickly look up the correct row + # when we iterate through the keys since they are in the correct order + values_dict = defaultdict(list) + for row in rows: + values_dict[row[0]].append(row[1:]) + + for key in keys: + if key in values_dict: + key_values = values_dict[key] + res = {} + res_ts: Optional[datetime] = None + for feature_name, value_bin, event_ts in key_values: + val = ValueProto() + val.ParseFromString(bytes(value_bin)) + res[feature_name] = val + res_ts = event_ts + result.append((res_ts, res)) + else: + result.append((None, None)) + return result + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ) -> None: + project = config.project + with self._get_cursor(config) as cur: + # We don't create any special state for the entities in this implementation. + for table in tables_to_keep: + cur.execute( + f"""CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), + feature_name VARCHAR(256), + value BLOB, + event_ts timestamp NULL DEFAULT NULL, + created_ts timestamp NULL DEFAULT NULL, + PRIMARY KEY(entity_key, feature_name), + INDEX {_table_id(project, table)}_ek (entity_key))""" + ) + + for table in tables_to_delete: + _drop_table_and_index(cur, project, table) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ) -> None: + project = config.project + with self._get_cursor(config) as cur: + for table in tables: + _drop_table_and_index(cur, project, table) + + +def _drop_table_and_index(cur: Cursor, project: str, table: FeatureView) -> None: + table_name = _table_id(project, table) + cur.execute(f"DROP INDEX {table_name}_ek ON {table_name};") + cur.execute(f"DROP TABLE IF EXISTS {table_name}") + + +def _table_id(project: str, table: FeatureView) -> str: + return f"{project}_{table.name}" + + +def _to_naive_utc(ts: datetime) -> datetime: + if ts.tzinfo is None: + return ts + else: + return ts.astimezone(pytz.utc).replace(tzinfo=None) diff --git a/sdk/python/feast/infra/online_stores/contrib/singlestore_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/singlestore_repo_configuration.py new file mode 100644 index 0000000000..2debe0f0ee --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/singlestore_repo_configuration.py @@ -0,0 +1,10 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.singlestore import ( + SingleStoreOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig(online_store_creator=SingleStoreOnlineStoreCreator), +] diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 99b90a09a5..f3c379020d 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -63,6 +63,7 @@ "ikv": "feast.infra.online_stores.contrib.ikv_online_store.ikv.IKVOnlineStore", "elasticsearch": "feast.infra.online_stores.contrib.elasticsearch.ElasticSearchOnlineStore", "remote": "feast.infra.online_stores.remote.RemoteOnlineStore", + "singlestore": "feast.infra.online_stores.contrib.singlestore_online_store.singlestore.SingleStoreOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 97bdfc159b..a0faf3d9ef 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -68,7 +68,9 @@ botocore==1.34.99 # moto # s3transfer build==1.2.1 - # via pip-tools + # via + # pip-tools + # singlestoredb cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 @@ -505,6 +507,8 @@ pandas==2.2.2 # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert +parsimonious==0.10.0 + # via singlestoredb parso==0.8.4 # via jedi parsy==2.1 @@ -610,6 +614,7 @@ pygments==2.18.0 pyjwt[crypto]==2.8.0 # via # msal + # singlestoredb # snowflake-connector-python pymssql==2.3.0 pymysql==1.1.1 @@ -705,6 +710,7 @@ requests==2.31.0 # msal # requests-oauthlib # responses + # singlestoredb # snowflake-connector-python # sphinx # trino @@ -745,8 +751,10 @@ setuptools==70.0.0 # grpcio-tools # kubernetes # pip-tools + # singlestoredb shellingham==1.5.4 # via typer +singlestoredb==1.3.1 six==1.16.0 # via # asttokens @@ -794,6 +802,8 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 +sqlparams==6.0.1 + # via singlestoredb stack-data==0.6.3 # via ipython starlette==0.37.2 @@ -821,6 +831,7 @@ tomli==2.0.1 # pip-tools # pytest # pytest-env + # singlestoredb tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 @@ -946,7 +957,9 @@ websockets==12.0 werkzeug==3.0.3 # via moto wheel==0.43.0 - # via pip-tools + # via + # pip-tools + # singlestoredb widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index f6db0af6bc..cea8cc22d0 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -64,7 +64,9 @@ botocore==1.34.99 # moto # s3transfer build==1.2.1 - # via pip-tools + # via + # pip-tools + # singlestoredb cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 @@ -496,6 +498,8 @@ pandas==2.2.2 # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert +parsimonious==0.10.0 + # via singlestoredb parso==0.8.4 # via jedi parsy==2.1 @@ -601,6 +605,7 @@ pygments==2.18.0 pyjwt[crypto]==2.8.0 # via # msal + # singlestoredb # snowflake-connector-python pymssql==2.3.0 pymysql==1.1.1 @@ -696,6 +701,7 @@ requests==2.31.0 # msal # requests-oauthlib # responses + # singlestoredb # snowflake-connector-python # sphinx # trino @@ -736,8 +742,10 @@ setuptools==70.0.0 # grpcio-tools # kubernetes # pip-tools + # singlestoredb shellingham==1.5.4 # via typer +singlestoredb==1.3.1 six==1.16.0 # via # asttokens @@ -785,6 +793,8 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 +sqlparams==6.0.1 + # via singlestoredb stack-data==0.6.3 # via ipython starlette==0.37.2 @@ -925,7 +935,9 @@ websockets==12.0 werkzeug==3.0.3 # via moto wheel==0.43.0 - # via pip-tools + # via + # pip-tools + # singlestoredb widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 135b65a0cc..d7df488a88 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -68,7 +68,9 @@ botocore==1.34.99 # moto # s3transfer build==1.2.1 - # via pip-tools + # via + # pip-tools + # singlestoredb cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.3 @@ -514,6 +516,8 @@ pandas==2.2.2 # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert +parsimonious==0.10.0 + # via singlestoredb parso==0.8.4 # via jedi parsy==2.1 @@ -619,6 +623,7 @@ pygments==2.18.0 pyjwt[crypto]==2.8.0 # via # msal + # singlestoredb # snowflake-connector-python pymssql==2.3.0 pymysql==1.1.1 @@ -714,6 +719,7 @@ requests==2.31.0 # msal # requests-oauthlib # responses + # singlestoredb # snowflake-connector-python # sphinx # trino @@ -756,8 +762,10 @@ setuptools==70.0.0 # grpcio-tools # kubernetes # pip-tools + # singlestoredb shellingham==1.5.4 # via typer +singlestoredb==1.3.1 six==1.16.0 # via # asttokens @@ -805,6 +813,8 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 +sqlparams==6.0.1 + # via singlestoredb stack-data==0.6.3 # via ipython starlette==0.37.2 @@ -832,6 +842,7 @@ tomli==2.0.1 # pip-tools # pytest # pytest-env + # singlestoredb tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 @@ -960,7 +971,9 @@ websockets==12.0 werkzeug==3.0.3 # via moto wheel==0.43.0 - # via pip-tools + # via + # pip-tools + # singlestoredb widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/singlestore.py b/sdk/python/tests/integration/feature_repos/universal/online_store/singlestore.py new file mode 100644 index 0000000000..d3a02421d0 --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/singlestore.py @@ -0,0 +1,43 @@ +import subprocess +import time +from typing import Dict + +from testcontainers.core.container import DockerContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class SingleStoreOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = ( + DockerContainer("ghcr.io/singlestore-labs/singlestoredb-dev:latest") + .with_exposed_ports(3306) + .with_env("USER", "root") + .with_env("ROOT_PASSWORD", "test") + # this license key is authorized solely for use in SingleStore Feast tests and is subject to strict usage restrictions + # if you want a free SingleStore license for your own use please visit https://www.singlestore.com/cloud-trial/ + .with_env( + "LICENSE_KEY", + "BGIxODZiYTg1YWUxYjRlODRhYzRjMGFmYTA1OTkxYzgyAAAAAAAAAAABAAAAAAAAACgwNQIZANx4NIXJ7CWvKYYb3wIyRXxBY7fdAnLeSwIYLy2Q0jA124GAkl04yuGrD59Zpv85DVYXAA==", + ) + ) + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + time.sleep(30) + exposed_port = self.container.get_exposed_port("3306") + command = f"mysql -uroot -ptest -P {exposed_port} -e 'CREATE DATABASE feast;'" + subprocess.run(command, shell=True, check=True) + return { + "type": "singlestore", + "user": "root", + "password": "test", + "database": "feast", + "port": exposed_port, + } + + def teardown(self): + self.container.stop() diff --git a/setup.py b/setup.py index f954f19898..cffd91a0c5 100644 --- a/setup.py +++ b/setup.py @@ -155,6 +155,8 @@ ELASTICSEARCH_REQUIRED = ["elasticsearch>=8.13.0"] +SINGLESTORE_REQUIRED = ["singlestoredb"] + CI_REQUIRED = ( [ "build", @@ -218,6 +220,7 @@ + DELTA_REQUIRED + ELASTICSEARCH_REQUIRED + SQLITE_VEC_REQUIRED + + SINGLESTORE_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -386,6 +389,7 @@ def run(self): "delta": DELTA_REQUIRED, "elasticsearch": ELASTICSEARCH_REQUIRED, "sqlite_vec": SQLITE_VEC_REQUIRED, + "singlestore": SINGLESTORE_REQUIRED, }, include_package_data=True, license="Apache", From 43e198f6945c5e868ade341309f2c5ca39ac563e Mon Sep 17 00:00:00 2001 From: "bdodla@expedia.com" <13788369+EXPEbdodla@users.noreply.github.com> Date: Fri, 28 Jun 2024 17:58:35 -0700 Subject: [PATCH 04/33] fix: CGO Memory leak issue in GO Feature server (#4291) --- go/internal/feast/server/http_server.go | 24 +++++++++++++++---- .../feast/transformation/transformation.go | 12 ++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/go/internal/feast/server/http_server.go b/go/internal/feast/server/http_server.go index 75cdbe9929..7ebab429e7 100644 --- a/go/internal/feast/server/http_server.go +++ b/go/internal/feast/server/http_server.go @@ -4,14 +4,16 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "time" + "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/feast/model" + "github.com/feast-dev/feast/go/internal/feast/onlineserving" "github.com/feast-dev/feast/go/internal/feast/server/logging" "github.com/feast-dev/feast/go/protos/feast/serving" prototypes "github.com/feast-dev/feast/go/protos/feast/types" "github.com/feast-dev/feast/go/types" - "net/http" - "time" ) type httpServer struct { @@ -210,6 +212,8 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { "results": results, } + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(response) if err != nil { @@ -217,8 +221,6 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { return } - w.Header().Set("Content-Type", "application/json") - if featureService != nil && featureService.LoggingConfig != nil && s.loggingService != nil { logger, err := s.loggingService.GetOrCreateLogger(featureService) if err != nil { @@ -250,11 +252,19 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { return } } + go releaseCGOMemory(featureVectors) +} + +func releaseCGOMemory(featureVectors []*onlineserving.FeatureVector) { + for _, vector := range featureVectors { + vector.Values.Release() + } } func (s *httpServer) Serve(host string, port int) error { s.server = &http.Server{Addr: fmt.Sprintf("%s:%d", host, port), Handler: nil} http.HandleFunc("/get-online-features", s.getOnlineFeatures) + http.HandleFunc("/health", healthCheckHandler) err := s.server.ListenAndServe() // Don't return the error if it's caused by graceful shutdown using Stop() if err == http.ErrServerClosed { @@ -262,6 +272,12 @@ func (s *httpServer) Serve(host string, port int) error { } return err } + +func healthCheckHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "Healthy") +} + func (s *httpServer) Stop() error { if s.server != nil { return s.server.Shutdown(context.Background()) diff --git a/go/internal/feast/transformation/transformation.go b/go/internal/feast/transformation/transformation.go index 1cf1dd3311..7e63aec224 100644 --- a/go/internal/feast/transformation/transformation.go +++ b/go/internal/feast/transformation/transformation.go @@ -46,6 +46,7 @@ func AugmentResponseWithOnDemandTransforms( for name, values := range requestData { requestContextArrow[name], err = types.ProtoValuesToArrowArray(values.Val, arrowMemory, numRows) if err != nil { + ReleaseArrowContext(requestContextArrow) return nil, err } } @@ -53,6 +54,7 @@ func AugmentResponseWithOnDemandTransforms( for name, values := range entityRows { requestContextArrow[name], err = types.ProtoValuesToArrowArray(values.Val, arrowMemory, numRows) if err != nil { + ReleaseArrowContext(requestContextArrow) return nil, err } } @@ -71,14 +73,24 @@ func AugmentResponseWithOnDemandTransforms( fullFeatureNames, ) if err != nil { + ReleaseArrowContext(requestContextArrow) return nil, err } result = append(result, onDemandFeatures...) + + ReleaseArrowContext(requestContextArrow) } return result, nil } +func ReleaseArrowContext(requestContextArrow map[string]arrow.Array) { + // Release memory used by requestContextArrow + for _, arrowArray := range requestContextArrow { + arrowArray.Release() + } +} + func CallTransformations( featureView *model.OnDemandFeatureView, retrievedFeatures map[string]arrow.Array, From 9451d9ca15f234e8e16e81351294fd63b33c1af2 Mon Sep 17 00:00:00 2001 From: Job Almekinders <55230856+job-almekinders@users.noreply.github.com> Date: Mon, 1 Jul 2024 23:04:59 +0200 Subject: [PATCH 05/33] feat: Bump psycopg2 to psycopg3 for all Postgres components (#4303) * Makefile: Formatting Signed-off-by: Job Almekinders * Makefile: Exclude Snowflake tests for postgres offline store tests Signed-off-by: Job Almekinders * Bootstrap: Use conninfo Signed-off-by: Job Almekinders * Tests: Make connection string compatible with psycopg3 Signed-off-by: Job Almekinders * Tests: Test connection type pool and singleton Signed-off-by: Job Almekinders * Global: Replace conn.set_session() calls to be psycopg3 compatible Set connection read only Signed-off-by: Job Almekinders * Offline: Use psycopg3 Signed-off-by: Job Almekinders * Online: Use psycopg3 Signed-off-by: Job Almekinders * Online: Restructure online_write_batch Addition Signed-off-by: Job Almekinders * Online: Use correct placeholder Signed-off-by: Job Almekinders * Online: Handle bytes properly in online_read() Signed-off-by: Job Almekinders * Online: Whitespace Signed-off-by: Job Almekinders * Online: Open ConnectionPool Signed-off-by: Job Almekinders * Online: Add typehint Signed-off-by: Job Almekinders * Utils: Use psycopg3 Use new ConnectionPool Pass kwargs as named argument Use executemany over execute_values Remove not-required open argument in psycopg.connect Improve Use SpooledTemporaryFile Use max_size and add docstring Properly write with StringIO Utils: Use SpooledTemporaryFile over StringIO object Add replace Fix df_to_postgres_table Remove import Utils Signed-off-by: Job Almekinders * Lint: Raise exceptions if cursor returned no columns or rows Add log statement Lint: Fix _to_arrow_internal Lint: Fix _get_entity_df_event_timestamp_range Update exception Use ZeroColumnQueryResult Signed-off-by: Job Almekinders * Add comment on +psycopg string Signed-off-by: Job Almekinders * Docs: Remove mention of psycopg2 Signed-off-by: Job Almekinders * Lint: Fix Signed-off-by: Job Almekinders * Default to postgresql+psycopg and log warning Update warning Fix Format warning Add typehints Use better variable name Signed-off-by: Job Almekinders * Solve merge conflicts Signed-off-by: Job Almekinders --------- Signed-off-by: Job Almekinders --- Makefile | 7 +- docs/tutorials/using-scalable-registry.md | 2 +- sdk/python/feast/errors.py | 10 + .../postgres_offline_store/postgres.py | 27 +- .../postgres_offline_store/postgres_source.py | 8 +- .../infra/online_stores/contrib/postgres.py | 120 +++---- .../infra/utils/postgres/connection_utils.py | 85 ++--- sdk/python/feast/repo_config.py | 16 + .../feast/templates/postgres/bootstrap.py | 16 +- .../requirements/py3.10-ci-requirements.txt | 297 ++++++++++++------ .../requirements/py3.10-requirements.txt | 48 ++- .../requirements/py3.11-ci-requirements.txt | 295 ++++++++++++----- .../requirements/py3.11-requirements.txt | 48 ++- .../requirements/py3.9-ci-requirements.txt | 291 ++++++++++++----- .../requirements/py3.9-requirements.txt | 48 ++- .../online_store/test_universal_online.py | 9 +- .../registration/test_universal_registry.py | 4 +- setup.py | 2 +- 18 files changed, 925 insertions(+), 408 deletions(-) diff --git a/Makefile b/Makefile index 39406cc17d..d2fbb34e1f 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ install-python: python setup.py develop lock-python-dependencies: - uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt + uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py$(PYTHON)-requirements.txt lock-python-dependencies-all: pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt" @@ -164,7 +164,7 @@ test-python-universal-mssql: sdk/python/tests -# To use Athena as an offline store, you need to create an Athena database and an S3 bucket on AWS. +# To use Athena as an offline store, you need to create an Athena database and an S3 bucket on AWS. # https://docs.aws.amazon.com/athena/latest/ug/getting-started.html # Modify environment variables ATHENA_REGION, ATHENA_DATA_SOURCE, ATHENA_DATABASE, ATHENA_WORKGROUP or # ATHENA_S3_BUCKET_NAME according to your needs. If tests fail with the pytest -n 8 option, change the number to 1. @@ -191,7 +191,7 @@ test-python-universal-athena: not s3_registry and \ not test_snowflake" \ sdk/python/tests - + test-python-universal-postgres-offline: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \ @@ -209,6 +209,7 @@ test-python-universal-postgres-offline: not test_push_features_to_offline_store and \ not gcs_registry and \ not s3_registry and \ + not test_snowflake and \ not test_universal_types" \ sdk/python/tests diff --git a/docs/tutorials/using-scalable-registry.md b/docs/tutorials/using-scalable-registry.md index 30b8e01ed5..25746f60e2 100644 --- a/docs/tutorials/using-scalable-registry.md +++ b/docs/tutorials/using-scalable-registry.md @@ -49,7 +49,7 @@ When this happens, your database is likely using what is referred to as an in `SQLAlchemy` terminology. See your database's documentation for examples on how to set its scheme in the Database URL. -`Psycopg2`, which is the database library leveraged by the online and offline +`Psycopg`, which is the database library leveraged by the online and offline stores, is not impacted by the need to speak a particular dialect, and so the following only applies to the registry. diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 6083b3d554..c4c1157626 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -389,3 +389,13 @@ def __init__(self, input_dict: dict): super().__init__( f"Failed to serialize the provided dictionary into a pandas DataFrame: {input_dict.keys()}" ) + + +class ZeroRowsQueryResult(Exception): + def __init__(self, query: str): + super().__init__(f"This query returned zero rows:\n{query}") + + +class ZeroColumnQueryResult(Exception): + def __init__(self, query: str): + super().__init__(f"This query returned zero columns:\n{query}") diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index cb08b5f016..c4740a960e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -19,11 +19,11 @@ import pandas as pd import pyarrow as pa from jinja2 import BaseLoader, Environment -from psycopg2 import sql +from psycopg import sql from pytz import utc from feast.data_source import DataSource -from feast.errors import InvalidEntityType +from feast.errors import InvalidEntityType, ZeroColumnQueryResult, ZeroRowsQueryResult from feast.feature_view import DUMMY_ENTITY_ID, DUMMY_ENTITY_VAL, FeatureView from feast.infra.offline_stores import offline_utils from feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source import ( @@ -274,8 +274,10 @@ def to_sql(self) -> str: def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: with self._query_generator() as query: with _get_conn(self.config.offline_store) as conn, conn.cursor() as cur: - conn.set_session(readonly=True) + conn.read_only = True cur.execute(query) + if not cur.description: + raise ZeroColumnQueryResult(query) fields = [ (c.name, pg_type_code_to_arrow(c.type_code)) for c in cur.description @@ -331,16 +333,19 @@ def _get_entity_df_event_timestamp_range( entity_df_event_timestamp.max().to_pydatetime(), ) elif isinstance(entity_df, str): - # If the entity_df is a string (SQL query), determine range - # from table + # If the entity_df is a string (SQL query), determine range from table with _get_conn(config.offline_store) as conn, conn.cursor() as cur: - ( - cur.execute( - f"SELECT MIN({entity_df_event_timestamp_col}) AS min, MAX({entity_df_event_timestamp_col}) AS max FROM ({entity_df}) as tmp_alias" - ), - ) + query = f""" + SELECT + MIN({entity_df_event_timestamp_col}) AS min, + MAX({entity_df_event_timestamp_col}) AS max + FROM ({entity_df}) AS tmp_alias + """ + cur.execute(query) res = cur.fetchone() - entity_df_event_timestamp_range = (res[0], res[1]) + if not res: + raise ZeroRowsQueryResult(query) + entity_df_event_timestamp_range = (res[0], res[1]) else: raise InvalidEntityType(type(entity_df)) diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py index bbb3f768fd..c216328b8d 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py @@ -4,7 +4,7 @@ from typeguard import typechecked from feast.data_source import DataSource -from feast.errors import DataSourceNoNameException +from feast.errors import DataSourceNoNameException, ZeroColumnQueryResult from feast.infra.utils.postgres.connection_utils import _get_conn from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.protos.feast.core.SavedDataset_pb2 import ( @@ -111,7 +111,11 @@ def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: with _get_conn(config.offline_store) as conn, conn.cursor() as cur: - cur.execute(f"SELECT * FROM {self.get_table_query_string()} AS sub LIMIT 0") + query = f"SELECT * FROM {self.get_table_query_string()} AS sub LIMIT 0" + cur.execute(query) + if not cur.description: + raise ZeroColumnQueryResult(query) + return ( (c.name, pg_type_code_to_pg_type(c.type_code)) for c in cur.description ) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 3eddd8ba20..8715f0f65b 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -2,13 +2,22 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Generator, + List, + Literal, + Optional, + Sequence, + Tuple, +) -import psycopg2 import pytz -from psycopg2 import sql -from psycopg2.extras import execute_values -from psycopg2.pool import SimpleConnectionPool +from psycopg import sql +from psycopg.connection import Connection +from psycopg_pool import ConnectionPool from feast import Entity from feast.feature_view import FeatureView @@ -39,15 +48,17 @@ class PostgreSQLOnlineStoreConfig(PostgreSQLConfig): class PostgreSQLOnlineStore(OnlineStore): - _conn: Optional[psycopg2._psycopg.connection] = None - _conn_pool: Optional[SimpleConnectionPool] = None + _conn: Optional[Connection] = None + _conn_pool: Optional[ConnectionPool] = None @contextlib.contextmanager - def _get_conn(self, config: RepoConfig): + def _get_conn(self, config: RepoConfig) -> Generator[Connection, Any, Any]: assert config.online_store.type == "postgres" + if config.online_store.conn_type == ConnectionType.pool: if not self._conn_pool: self._conn_pool = _get_connection_pool(config.online_store) + self._conn_pool.open() connection = self._conn_pool.getconn() yield connection self._conn_pool.putconn(connection) @@ -64,57 +75,56 @@ def online_write_batch( Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] ], progress: Optional[Callable[[int], Any]], + batch_size: int = 5000, ) -> None: - project = config.project + # Format insert values + insert_values = [] + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) - with self._get_conn(config) as conn, conn.cursor() as cur: - insert_values = [] - for entity_key, values, timestamp, created_ts in data: - entity_key_bin = serialize_entity_key( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) - timestamp = _to_naive_utc(timestamp) - if created_ts is not None: - created_ts = _to_naive_utc(created_ts) - - for feature_name, val in values.items(): - vector_val = None - if config.online_store.pgvector_enabled: - vector_val = get_list_val_str(val) - insert_values.append( - ( - entity_key_bin, - feature_name, - val.SerializeToString(), - vector_val, - timestamp, - created_ts, - ) + for feature_name, val in values.items(): + vector_val = None + if config.online_store.pgvector_enabled: + vector_val = get_list_val_str(val) + insert_values.append( + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + vector_val, + timestamp, + created_ts, ) - # Control the batch so that we can update the progress - batch_size = 5000 + ) + + # Create insert query + sql_query = sql.SQL( + """ + INSERT INTO {} + (entity_key, feature_name, value, vector_value, event_ts, created_ts) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (entity_key, feature_name) DO + UPDATE SET + value = EXCLUDED.value, + vector_value = EXCLUDED.vector_value, + event_ts = EXCLUDED.event_ts, + created_ts = EXCLUDED.created_ts; + """ + ).format(sql.Identifier(_table_id(config.project, table))) + + # Push data in batches to online store + with self._get_conn(config) as conn, conn.cursor() as cur: for i in range(0, len(insert_values), batch_size): cur_batch = insert_values[i : i + batch_size] - execute_values( - cur, - sql.SQL( - """ - INSERT INTO {} - (entity_key, feature_name, value, vector_value, event_ts, created_ts) - VALUES %s - ON CONFLICT (entity_key, feature_name) DO - UPDATE SET - value = EXCLUDED.value, - vector_value = EXCLUDED.vector_value, - event_ts = EXCLUDED.event_ts, - created_ts = EXCLUDED.created_ts; - """, - ).format(sql.Identifier(_table_id(project, table))), - cur_batch, - page_size=batch_size, - ) + cur.executemany(sql_query, cur_batch) conn.commit() + if progress: progress(len(cur_batch)) @@ -172,7 +182,9 @@ def online_read( # when we iterate through the keys since they are in the correct order values_dict = defaultdict(list) for row in rows if rows is not None else []: - values_dict[row[0].tobytes()].append(row[1:]) + values_dict[ + row[0] if isinstance(row[0], bytes) else row[0].tobytes() + ].append(row[1:]) for key in keys: if key in values_dict: diff --git a/sdk/python/feast/infra/utils/postgres/connection_utils.py b/sdk/python/feast/infra/utils/postgres/connection_utils.py index 0d99c8ab99..e0599019b9 100644 --- a/sdk/python/feast/infra/utils/postgres/connection_utils.py +++ b/sdk/python/feast/infra/utils/postgres/connection_utils.py @@ -1,50 +1,59 @@ -from typing import Dict +from typing import Any, Dict import numpy as np import pandas as pd -import psycopg2 -import psycopg2.extras +import psycopg import pyarrow as pa -from psycopg2.pool import SimpleConnectionPool +from psycopg.connection import Connection +from psycopg_pool import ConnectionPool from feast.infra.utils.postgres.postgres_config import PostgreSQLConfig from feast.type_map import arrow_to_pg_type -def _get_conn(config: PostgreSQLConfig): - conn = psycopg2.connect( - dbname=config.database, - host=config.host, - port=int(config.port), - user=config.user, - password=config.password, - sslmode=config.sslmode, - sslkey=config.sslkey_path, - sslcert=config.sslcert_path, - sslrootcert=config.sslrootcert_path, - options="-c search_path={}".format(config.db_schema or config.user), +def _get_conn(config: PostgreSQLConfig) -> Connection: + """Get a psycopg `Connection`.""" + conn = psycopg.connect( + conninfo=_get_conninfo(config), keepalives_idle=config.keepalives_idle, + **_get_conn_kwargs(config), ) return conn -def _get_connection_pool(config: PostgreSQLConfig): - return SimpleConnectionPool( - config.min_conn, - config.max_conn, - dbname=config.database, - host=config.host, - port=int(config.port), - user=config.user, - password=config.password, - sslmode=config.sslmode, - sslkey=config.sslkey_path, - sslcert=config.sslcert_path, - sslrootcert=config.sslrootcert_path, - options="-c search_path={}".format(config.db_schema or config.user), +def _get_connection_pool(config: PostgreSQLConfig) -> ConnectionPool: + """Get a psycopg `ConnectionPool`.""" + return ConnectionPool( + conninfo=_get_conninfo(config), + min_size=config.min_conn, + max_size=config.max_conn, + open=False, + kwargs=_get_conn_kwargs(config), ) +def _get_conninfo(config: PostgreSQLConfig) -> str: + """Get the `conninfo` argument required for connection objects.""" + return ( + f"postgresql://{config.user}" + f":{config.password}" + f"@{config.host}" + f":{int(config.port)}" + f"/{config.database}" + ) + + +def _get_conn_kwargs(config: PostgreSQLConfig) -> Dict[str, Any]: + """Get the additional `kwargs` required for connection objects.""" + return { + "sslmode": config.sslmode, + "sslkey": config.sslkey_path, + "sslcert": config.sslcert_path, + "sslrootcert": config.sslrootcert_path, + "options": "-c search_path={}".format(config.db_schema or config.user), + } + + def _df_to_create_table_sql(entity_df, table_name) -> str: pa_table = pa.Table.from_pandas(entity_df) columns = [ @@ -63,16 +72,14 @@ def df_to_postgres_table( """ Create a table for the data frame, insert all the values, and return the table schema """ + nr_columns = df.shape[1] + placeholders = ", ".join(["%s"] * nr_columns) + query = f"INSERT INTO {table_name} VALUES ({placeholders})" + values = df.replace({np.NaN: None}).to_numpy().tolist() + with _get_conn(config) as conn, conn.cursor() as cur: cur.execute(_df_to_create_table_sql(df, table_name)) - psycopg2.extras.execute_values( - cur, - f""" - INSERT INTO {table_name} - VALUES %s - """, - df.replace({np.NaN: None}).to_numpy(), - ) + cur.executemany(query, values) return dict(zip(df.columns, df.dtypes)) @@ -82,7 +89,7 @@ def get_query_schema(config: PostgreSQLConfig, sql_query: str) -> Dict[str, np.d new table """ with _get_conn(config) as conn: - conn.set_session(readonly=True) + conn.read_only = True df = pd.read_sql( f"SELECT * FROM {sql_query} LIMIT 0", conn, diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index f3c379020d..8d6bff2818 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -12,6 +12,7 @@ StrictInt, StrictStr, ValidationError, + ValidationInfo, field_validator, model_validator, ) @@ -123,6 +124,21 @@ class RegistryConfig(FeastBaseModel): sqlalchemy_config_kwargs: Dict[str, Any] = {} """ Dict[str, Any]: Extra arguments to pass to SQLAlchemy.create_engine. """ + @field_validator("path") + def validate_path(cls, path: str, values: ValidationInfo) -> str: + if values.data.get("registry_type") == "sql": + if path.startswith("postgresql://"): + _logger.warning( + "The `path` of the `RegistryConfig` starts with a plain " + "`postgresql` string. We are updating this to `postgresql+psycopg` " + "to ensure that the `psycopg3` driver is used by `sqlalchemy`. If " + "you want to use `psycopg2` pass `postgresql+psycopg2` explicitely " + "to `path`. To silence this warning, pass `postgresql+psycopg` " + "explicitely to `path`." + ) + return path.replace("postgresql://", "postgresql+psycopg://") + return path + class RepoConfig(FeastBaseModel): """Repo config. Typically loaded from `feature_store.yaml`""" diff --git a/sdk/python/feast/templates/postgres/bootstrap.py b/sdk/python/feast/templates/postgres/bootstrap.py index 9f6e8a988d..6ed13e4e39 100644 --- a/sdk/python/feast/templates/postgres/bootstrap.py +++ b/sdk/python/feast/templates/postgres/bootstrap.py @@ -1,5 +1,5 @@ import click -import psycopg2 +import psycopg from feast.file_utils import replace_str_in_file from feast.infra.utils.postgres.connection_utils import df_to_postgres_table @@ -34,12 +34,14 @@ def bootstrap(): 'Should I upload example data to Postgres (overwriting "feast_driver_hourly_stats" table)?', default=True, ): - db_connection = psycopg2.connect( - dbname=postgres_database, - host=postgres_host, - port=int(postgres_port), - user=postgres_user, - password=postgres_password, + db_connection = psycopg.connect( + conninfo=( + f"postgresql://{postgres_user}" + f":{postgres_password}" + f"@{postgres_host}" + f":{int(postgres_port)}" + f"/{postgres_database}" + ), options=f"-c search_path={postgres_schema}", ) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index a0faf3d9ef..3aa7130ccf 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt -aiobotocore==2.13.0 +aiobotocore==2.13.1 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -11,14 +12,16 @@ alabaster==0.7.16 # via sphinx altair==4.2.2 # via great-expectations -annotated-types==0.6.0 +annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.4.0 # via # httpx # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -43,12 +47,14 @@ attrs==23.2.0 # aiohttp # jsonschema # referencing -azure-core==1.30.1 +azure-core==1.30.2 # via # azure-identity # azure-storage-blob -azure-identity==1.16.0 +azure-identity==1.17.1 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -59,9 +65,11 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.99 - # via moto -botocore==1.34.99 +boto3==1.34.131 + # via + # feast (setup.py) + # moto +botocore==1.34.131 # via # aiobotocore # boto3 @@ -69,6 +77,7 @@ botocore==1.34.99 # s3transfer build==1.2.1 # via + # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -76,7 +85,8 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 -certifi==2024.2.2 + # via feast (setup.py) +certifi==2024.6.2 # via # elastic-transport # httpcore @@ -98,6 +108,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -107,15 +118,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.3 +coverage[toml]==7.5.4 # via pytest-cov -cryptography==42.0.7 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -126,20 +140,24 @@ cryptography==42.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask[dataframe]==2024.5.0 - # via dask-expr -dask-expr==1.1.0 +dask[dataframe]==2024.6.2 + # via + # feast (setup.py) + # dask-expr +dask-expr==1.1.6 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.1 +debugpy==1.8.2 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.3 +deltalake==0.18.1 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -152,12 +170,13 @@ duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.1 +duckdb-engine==0.13.0 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch -elasticsearch==8.13.2 -email-validator==2.1.1 +elasticsearch==8.14.0 + # via feast (setup.py) +email-validator==2.2.0 # via fastapi entrypoints==0.4 # via altair @@ -171,16 +190,17 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via fastapi-cli -fastapi-cli==0.0.2 + # via feast (setup.py) +fastapi-cli==0.0.4 # via fastapi -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat -filelock==3.14.0 +filelock==3.15.4 # via # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -188,13 +208,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.19.0 +google-api-core[grpc]==2.19.1 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -204,9 +227,9 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.131.0 +google-api-python-client==2.134.0 # via firebase-admin -google-auth==2.29.0 +google-auth==2.30.0 # via # google-api-core # google-api-python-client @@ -219,8 +242,11 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 -google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-bigtable==2.24.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -229,30 +255,34 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin -google-cloud-storage==2.16.0 - # via firebase-admin +google-cloud-storage==2.17.0 + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.7.0 +google-resumable-media==2.7.1 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 +googleapis-common-protos[grpc]==1.63.2 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.15 -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 +great-expectations==0.18.16 + # via feast (setup.py) +grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.64.0 +grpcio==1.64.1 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -263,19 +293,27 @@ grpcio==1.64.0 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -286,11 +324,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==3.2.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -304,7 +346,7 @@ idna==3.7 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==7.1.0 +importlib-metadata==8.0.0 # via dask iniconfig==2.0.0 # via pytest @@ -325,6 +367,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -342,12 +385,13 @@ json5==0.9.25 # via jupyterlab-server jsonpatch==1.33 # via great-expectations -jsonpointer==2.4 +jsonpointer==3.0.0 # via # jsonpatch # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -382,7 +426,7 @@ jupyter-server==2.14.1 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.1 +jupyterlab==4.2.3 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -393,6 +437,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -404,7 +449,7 @@ markupsafe==2.1.5 # jinja2 # nbconvert # werkzeug -marshmallow==3.21.2 +marshmallow==3.21.3 # via great-expectations matplotlib-inline==0.1.7 # via @@ -413,18 +458,22 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 -msal==1.28.0 + # via feast (setup.py) +msal==1.29.0 # via # azure-identity # msal-extensions -msal-extensions==1.1.0 +msal-extensions==1.2.0 # via azure-identity msgpack==1.0.8 # via cachecontrol @@ -434,11 +483,14 @@ multidict==6.0.5 # yarl multipledispatch==1.0.0 # via ibis-framework -mypy==1.10.0 - # via sqlalchemy +mypy==1.10.1 + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -451,9 +503,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.9.0 +nodeenv==1.9.1 # via pre-commit -notebook==7.2.0 +notebook==7.2.1 # via great-expectations notebook-shim==0.2.4 # via @@ -461,6 +513,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -471,11 +524,11 @@ numpy==1.26.4 # scipy oauthlib==3.2.2 # via requests-oauthlib -orjson==3.10.3 +orjson==3.10.5 # via fastapi overrides==7.7.0 # via jupyter-server -packaging==24.0 +packaging==24.1 # via # build # dask @@ -490,13 +543,13 @@ packaging==24.0 # jupyterlab # jupyterlab-server # marshmallow - # msal-extensions # nbconvert # pytest # snowflake-connector-python # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -519,9 +572,10 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip==24.0 +pip==24.1.1 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -531,14 +585,15 @@ pluggy==1.5.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.8.2 +portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.45 +prompt-toolkit==3.0.47 # via ipython -proto-plus==1.23.0 +proto-plus==1.24.0 # via # google-api-core # google-cloud-bigquery @@ -548,6 +603,7 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -565,8 +621,15 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 + # via psycopg +psycopg-pool==3.2.2 + # via psycopg ptyprocess==0.7.0 # via # pexpect @@ -574,12 +637,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -597,16 +662,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.7.1 +pydantic==2.7.4 # via + # feast (setup.py) # fastapi # great-expectations -pydantic-core==2.18.2 +pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -617,8 +685,11 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -630,8 +701,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -641,13 +714,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -676,6 +757,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -689,14 +771,19 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.28 -requests==2.31.0 +regex==2024.5.15 # via + # feast (setup.py) + # parsimonious +requests==2.32.3 + # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -716,7 +803,7 @@ requests==2.31.0 # trino requests-oauthlib==2.0.0 # via kubernetes -responses==0.25.0 +responses==0.25.3 # via moto rfc3339-validator==0.1.4 # via @@ -731,6 +818,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -739,22 +827,25 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.6 -s3transfer==0.10.1 +ruff==0.4.10 + # via feast (setup.py) +s3transfer==0.10.2 # via boto3 -scipy==1.13.1 +scipy==1.14.0 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==70.0.0 +setuptools==70.1.1 # via # grpcio-tools + # jupyterlab # kubernetes # pip-tools # singlestoredb shellingham==1.5.4 # via typer -singlestoredb==1.3.1 +singlestoredb==1.4.0 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -774,12 +865,14 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.1 +snowflake-connector-python[pandas]==3.11.0 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -792,8 +885,9 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.30 +sqlalchemy[mypy]==2.0.31 # via + # feast (setup.py) # duckdb-engine # ibis-framework # sqlalchemy-views @@ -802,6 +896,7 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -811,17 +906,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 -tenacity==8.3.0 + # via feast (setup.py) +tenacity==8.4.2 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 -thriftpy2==0.5.0 + # via feast (setup.py) +thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -849,7 +948,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -866,38 +967,55 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 -typeguard==4.2.1 + # via feast (setup.py) +typeguard==4.3.0 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf -types-pymysql==1.1.0.20240425 + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 -types-setuptools==70.0.0.20240524 - # via types-cffi + # via feast (setup.py) +types-setuptools==70.1.0.20240627 + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via # anyio # async-lru # azure-core + # azure-identity # azure-storage-blob # fastapi # great-expectations # ibis-framework # ipython # mypy + # psycopg + # psycopg-pool # pydantic # pydantic-core # snowflake-connector-python @@ -912,14 +1030,15 @@ tzlocal==5.2 # via # great-expectations # trino -ujson==5.9.0 +ujson==5.10.0 # via fastapi uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.18 +urllib3==1.26.19 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -930,19 +1049,21 @@ urllib3==1.26.18 # responses # rockset # testcontainers -uvicorn[standard]==0.29.0 +uvicorn[standard]==0.30.1 # via + # feast (setup.py) # fastapi - # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit -watchfiles==0.21.0 + # via + # feast (setup.py) + # pre-commit +watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit -webcolors==1.13 +webcolors==24.6.0 # via jsonschema webencodings==0.5.1 # via @@ -970,5 +1091,5 @@ xmltodict==0.13.0 # via moto yarl==1.9.4 # via aiohttp -zipp==3.18.1 +zipp==3.19.2 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 99c9bfc3fe..72124636b6 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -65,8 +71,11 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -78,13 +87,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -96,20 +108,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -120,6 +141,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -127,6 +149,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -142,11 +165,15 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -154,7 +181,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -178,6 +207,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index cea8cc22d0..673047b5c7 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt -aiobotocore==2.13.0 +aiobotocore==2.13.1 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -11,14 +12,16 @@ alabaster==0.7.16 # via sphinx altair==4.2.2 # via great-expectations -annotated-types==0.6.0 +annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.4.0 # via # httpx # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -39,12 +43,14 @@ attrs==23.2.0 # aiohttp # jsonschema # referencing -azure-core==1.30.1 +azure-core==1.30.2 # via # azure-identity # azure-storage-blob -azure-identity==1.16.0 +azure-identity==1.17.1 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -55,9 +61,11 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.99 - # via moto -botocore==1.34.99 +boto3==1.34.131 + # via + # feast (setup.py) + # moto +botocore==1.34.131 # via # aiobotocore # boto3 @@ -65,6 +73,7 @@ botocore==1.34.99 # s3transfer build==1.2.1 # via + # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -72,7 +81,8 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 -certifi==2024.2.2 + # via feast (setup.py) +certifi==2024.6.2 # via # elastic-transport # httpcore @@ -94,6 +104,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -103,15 +114,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.3 +coverage[toml]==7.5.4 # via pytest-cov -cryptography==42.0.7 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -122,20 +136,24 @@ cryptography==42.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask[dataframe]==2024.5.0 - # via dask-expr -dask-expr==1.1.0 +dask[dataframe]==2024.6.2 + # via + # feast (setup.py) + # dask-expr +dask-expr==1.1.6 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.1 +debugpy==1.8.2 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.4 +deltalake==0.18.1 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -148,12 +166,13 @@ duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.1 +duckdb-engine==0.13.0 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch -elasticsearch==8.13.2 -email-validator==2.1.1 +elasticsearch==8.14.0 + # via feast (setup.py) +email-validator==2.2.0 # via fastapi entrypoints==0.4 # via altair @@ -162,16 +181,17 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via fastapi-cli -fastapi-cli==0.0.2 + # via feast (setup.py) +fastapi-cli==0.0.4 # via fastapi -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat -filelock==3.14.0 +filelock==3.15.4 # via # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -179,13 +199,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.19.0 +google-api-core[grpc]==2.19.1 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -195,9 +218,9 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.131.0 +google-api-python-client==2.134.0 # via firebase-admin -google-auth==2.29.0 +google-auth==2.30.0 # via # google-api-core # google-api-python-client @@ -210,8 +233,11 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 -google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-bigtable==2.24.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -220,30 +246,34 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin -google-cloud-storage==2.16.0 - # via firebase-admin +google-cloud-storage==2.17.0 + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.7.0 +google-resumable-media==2.7.1 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 +googleapis-common-protos[grpc]==1.63.2 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.15 -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 +great-expectations==0.18.16 + # via feast (setup.py) +grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.64.0 +grpcio==1.64.1 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -254,19 +284,27 @@ grpcio==1.64.0 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -277,11 +315,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==3.2.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -295,7 +337,7 @@ idna==3.7 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==7.1.0 +importlib-metadata==8.0.0 # via dask iniconfig==2.0.0 # via pytest @@ -316,6 +358,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -333,12 +376,13 @@ json5==0.9.25 # via jupyterlab-server jsonpatch==1.33 # via great-expectations -jsonpointer==2.4 +jsonpointer==3.0.0 # via # jsonpatch # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -373,7 +417,7 @@ jupyter-server==2.14.1 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.1 +jupyterlab==4.2.3 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -384,6 +428,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -395,7 +440,7 @@ markupsafe==2.1.5 # jinja2 # nbconvert # werkzeug -marshmallow==3.21.2 +marshmallow==3.21.3 # via great-expectations matplotlib-inline==0.1.7 # via @@ -404,18 +449,22 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 -msal==1.28.0 + # via feast (setup.py) +msal==1.29.0 # via # azure-identity # msal-extensions -msal-extensions==1.1.0 +msal-extensions==1.2.0 # via azure-identity msgpack==1.0.8 # via cachecontrol @@ -425,11 +474,14 @@ multidict==6.0.5 # yarl multipledispatch==1.0.0 # via ibis-framework -mypy==1.10.0 - # via sqlalchemy +mypy==1.10.1 + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -442,9 +494,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.9.0 +nodeenv==1.9.1 # via pre-commit -notebook==7.2.0 +notebook==7.2.1 # via great-expectations notebook-shim==0.2.4 # via @@ -452,6 +504,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -462,11 +515,11 @@ numpy==1.26.4 # scipy oauthlib==3.2.2 # via requests-oauthlib -orjson==3.10.3 +orjson==3.10.5 # via fastapi overrides==7.7.0 # via jupyter-server -packaging==24.0 +packaging==24.1 # via # build # dask @@ -481,13 +534,13 @@ packaging==24.0 # jupyterlab # jupyterlab-server # marshmallow - # msal-extensions # nbconvert # pytest # snowflake-connector-python # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -510,9 +563,10 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip==24.0 +pip==24.1.1 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -522,14 +576,15 @@ pluggy==1.5.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.8.2 +portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.45 +prompt-toolkit==3.0.47 # via ipython -proto-plus==1.23.0 +proto-plus==1.24.0 # via # google-api-core # google-cloud-bigquery @@ -539,6 +594,7 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -556,8 +612,15 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 + # via psycopg +psycopg-pool==3.2.2 + # via psycopg ptyprocess==0.7.0 # via # pexpect @@ -565,12 +628,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -588,16 +653,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.7.1 +pydantic==2.7.4 # via + # feast (setup.py) # fastapi # great-expectations -pydantic-core==2.18.2 +pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -608,8 +676,11 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -621,8 +692,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -632,13 +705,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -667,6 +748,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -680,14 +762,19 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 -requests==2.31.0 # via + # feast (setup.py) + # parsimonious +requests==2.32.3 + # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -707,7 +794,7 @@ requests==2.31.0 # trino requests-oauthlib==2.0.0 # via kubernetes -responses==0.25.0 +responses==0.25.3 # via moto rfc3339-validator==0.1.4 # via @@ -722,6 +809,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -730,22 +818,25 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.6 -s3transfer==0.10.1 +ruff==0.4.10 + # via feast (setup.py) +s3transfer==0.10.2 # via boto3 -scipy==1.13.1 +scipy==1.14.0 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==70.0.0 +setuptools==70.1.1 # via # grpcio-tools + # jupyterlab # kubernetes # pip-tools # singlestoredb shellingham==1.5.4 # via typer -singlestoredb==1.3.1 +singlestoredb==1.4.0 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -765,12 +856,14 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.1 +snowflake-connector-python[pandas]==3.11.0 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -783,8 +876,9 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.30 +sqlalchemy[mypy]==2.0.31 # via + # feast (setup.py) # duckdb-engine # ibis-framework # sqlalchemy-views @@ -793,6 +887,7 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -802,17 +897,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 -tenacity==8.3.0 + # via feast (setup.py) +tenacity==8.4.2 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 -thriftpy2==0.5.0 + # via feast (setup.py) +thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 @@ -830,7 +929,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -847,36 +948,53 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 -typeguard==4.2.1 + # via feast (setup.py) +typeguard==4.3.0 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf -types-pymysql==1.1.0.20240425 + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 -types-setuptools==70.0.0.20240524 - # via types-cffi + # via feast (setup.py) +types-setuptools==70.1.0.20240627 + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via # azure-core + # azure-identity # azure-storage-blob # fastapi # great-expectations # ibis-framework # ipython # mypy + # psycopg + # psycopg-pool # pydantic # pydantic-core # snowflake-connector-python @@ -890,14 +1008,15 @@ tzlocal==5.2 # via # great-expectations # trino -ujson==5.9.0 +ujson==5.10.0 # via fastapi uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.18 +urllib3==1.26.19 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -908,19 +1027,21 @@ urllib3==1.26.18 # responses # rockset # testcontainers -uvicorn[standard]==0.29.0 +uvicorn[standard]==0.30.1 # via + # feast (setup.py) # fastapi - # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit -watchfiles==0.21.0 + # via + # feast (setup.py) + # pre-commit +watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit -webcolors==1.13 +webcolors==24.6.0 # via jsonschema webencodings==0.5.1 # via @@ -948,5 +1069,5 @@ xmltodict==0.13.0 # via moto yarl==1.9.4 # via aiohttp -zipp==3.18.1 +zipp==3.19.2 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index c34b610d14..408f392515 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -20,30 +20,36 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -63,8 +69,11 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -76,13 +85,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -94,20 +106,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -118,6 +139,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -125,6 +147,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -140,17 +163,23 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) toolz==0.12.1 # via # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -172,6 +201,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index d7df488a88..83009f8730 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt -aiobotocore==2.13.0 +aiobotocore==2.13.1 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -11,14 +12,16 @@ alabaster==0.7.16 # via sphinx altair==4.2.2 # via great-expectations -annotated-types==0.6.0 +annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.4.0 # via # httpx # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -43,12 +47,14 @@ attrs==23.2.0 # aiohttp # jsonschema # referencing -azure-core==1.30.1 +azure-core==1.30.2 # via # azure-identity # azure-storage-blob -azure-identity==1.16.0 +azure-identity==1.17.1 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -59,9 +65,11 @@ bidict==0.23.1 # via ibis-framework bleach==6.1.0 # via nbconvert -boto3==1.34.99 - # via moto -botocore==1.34.99 +boto3==1.34.131 + # via + # feast (setup.py) + # moto +botocore==1.34.131 # via # aiobotocore # boto3 @@ -69,6 +77,7 @@ botocore==1.34.99 # s3transfer build==1.2.1 # via + # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -76,7 +85,8 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 -certifi==2024.2.2 + # via feast (setup.py) +certifi==2024.6.2 # via # elastic-transport # httpcore @@ -98,6 +108,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -107,15 +118,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.3 +coverage[toml]==7.5.4 # via pytest-cov -cryptography==42.0.7 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -126,20 +140,24 @@ cryptography==42.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask[dataframe]==2024.5.0 - # via dask-expr -dask-expr==1.1.0 +dask[dataframe]==2024.6.2 + # via + # feast (setup.py) + # dask-expr +dask-expr==1.1.6 # via dask db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.1 +debugpy==1.8.2 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.4 +deltalake==0.18.1 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -152,12 +170,13 @@ duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.1 +duckdb-engine==0.13.0 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch -elasticsearch==8.13.2 -email-validator==2.1.1 +elasticsearch==8.14.0 + # via feast (setup.py) +email-validator==2.2.0 # via fastapi entrypoints==0.4 # via altair @@ -171,16 +190,17 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via fastapi-cli -fastapi-cli==0.0.2 + # via feast (setup.py) +fastapi-cli==0.0.4 # via fastapi -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat -filelock==3.14.0 +filelock==3.15.4 # via # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -188,13 +208,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.19.0 +google-api-core[grpc]==2.19.1 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -204,9 +227,9 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.131.0 +google-api-python-client==2.134.0 # via firebase-admin -google-auth==2.29.0 +google-auth==2.30.0 # via # google-api-core # google-api-python-client @@ -219,8 +242,11 @@ google-auth==2.29.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 -google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-bigtable==2.24.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -229,30 +255,34 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin -google-cloud-storage==2.16.0 - # via firebase-admin +google-cloud-storage==2.17.0 + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.7.0 +google-resumable-media==2.7.1 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 +googleapis-common-protos[grpc]==1.63.2 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.15 -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 +great-expectations==0.18.16 + # via feast (setup.py) +grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.64.0 +grpcio==1.64.1 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -263,19 +293,27 @@ grpcio==1.64.0 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -286,11 +324,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==8.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==3.2.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -304,7 +346,7 @@ idna==3.7 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==7.1.0 +importlib-metadata==8.0.0 # via # build # dask @@ -334,6 +376,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -351,12 +394,13 @@ json5==0.9.25 # via jupyterlab-server jsonpatch==1.33 # via great-expectations -jsonpointer==2.4 +jsonpointer==3.0.0 # via # jsonpatch # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -391,7 +435,7 @@ jupyter-server==2.14.1 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.1 +jupyterlab==4.2.3 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -402,6 +446,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -413,7 +458,7 @@ markupsafe==2.1.5 # jinja2 # nbconvert # werkzeug -marshmallow==3.21.2 +marshmallow==3.21.3 # via great-expectations matplotlib-inline==0.1.7 # via @@ -422,18 +467,22 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 -msal==1.28.0 + # via feast (setup.py) +msal==1.29.0 # via # azure-identity # msal-extensions -msal-extensions==1.1.0 +msal-extensions==1.2.0 # via azure-identity msgpack==1.0.8 # via cachecontrol @@ -443,11 +492,14 @@ multidict==6.0.5 # yarl multipledispatch==1.0.0 # via ibis-framework -mypy==1.10.0 - # via sqlalchemy +mypy==1.10.1 + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -460,9 +512,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.9.0 +nodeenv==1.9.1 # via pre-commit -notebook==7.2.0 +notebook==7.2.1 # via great-expectations notebook-shim==0.2.4 # via @@ -470,6 +522,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -480,11 +533,11 @@ numpy==1.26.4 # scipy oauthlib==3.2.2 # via requests-oauthlib -orjson==3.10.3 +orjson==3.10.5 # via fastapi overrides==7.7.0 # via jupyter-server -packaging==24.0 +packaging==24.1 # via # build # dask @@ -499,13 +552,13 @@ packaging==24.0 # jupyterlab # jupyterlab-server # marshmallow - # msal-extensions # nbconvert # pytest # snowflake-connector-python # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -528,9 +581,10 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip==24.0 +pip==24.1.1 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -540,14 +594,15 @@ pluggy==1.5.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.8.2 +portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.45 +prompt-toolkit==3.0.47 # via ipython -proto-plus==1.23.0 +proto-plus==1.24.0 # via # google-api-core # google-cloud-bigquery @@ -557,6 +612,7 @@ proto-plus==1.23.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -574,8 +630,15 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.18 + # via feast (setup.py) +psycopg-binary==3.1.18 + # via psycopg +psycopg-pool==3.2.2 + # via psycopg ptyprocess==0.7.0 # via # pexpect @@ -583,12 +646,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -606,16 +671,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.7.1 +pydantic==2.7.4 # via + # feast (setup.py) # fastapi # great-expectations -pydantic-core==2.18.2 +pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -626,8 +694,11 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -639,8 +710,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -650,13 +723,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -685,6 +766,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -698,14 +780,19 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 -requests==2.31.0 # via + # feast (setup.py) + # parsimonious +requests==2.32.3 + # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -725,7 +812,7 @@ requests==2.31.0 # trino requests-oauthlib==2.0.0 # via kubernetes -responses==0.25.0 +responses==0.25.3 # via moto rfc3339-validator==0.1.4 # via @@ -740,6 +827,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -750,22 +838,25 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.4.6 -s3transfer==0.10.1 +ruff==0.4.10 + # via feast (setup.py) +s3transfer==0.10.2 # via boto3 scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==70.0.0 +setuptools==70.1.1 # via # grpcio-tools + # jupyterlab # kubernetes # pip-tools # singlestoredb shellingham==1.5.4 # via typer -singlestoredb==1.3.1 +singlestoredb==1.4.0 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -785,12 +876,14 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.1 +snowflake-connector-python[pandas]==3.11.0 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -803,8 +896,9 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==2.0.30 +sqlalchemy[mypy]==2.0.31 # via + # feast (setup.py) # duckdb-engine # ibis-framework # sqlalchemy-views @@ -813,6 +907,7 @@ sqlalchemy-views==0.3.2 sqlglot==20.11.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -822,17 +917,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 -tenacity==8.3.0 + # via feast (setup.py) +tenacity==8.4.2 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 -thriftpy2==0.5.0 + # via feast (setup.py) +thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -860,7 +959,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -877,39 +978,56 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 -typeguard==4.2.1 + # via feast (setup.py) +typeguard==4.3.0 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 -types-setuptools==70.0.0.20240524 - # via types-cffi + # via feast (setup.py) +types-setuptools==70.1.0.20240627 + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via # aioitertools # anyio # async-lru # azure-core + # azure-identity # azure-storage-blob # fastapi # great-expectations # ibis-framework # ipython # mypy + # psycopg + # psycopg-pool # pydantic # pydantic-core # snowflake-connector-python @@ -925,14 +1043,15 @@ tzlocal==5.2 # via # great-expectations # trino -ujson==5.9.0 +ujson==5.10.0 # via fastapi uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.18 +urllib3==1.26.19 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -944,19 +1063,21 @@ urllib3==1.26.18 # rockset # snowflake-connector-python # testcontainers -uvicorn[standard]==0.29.0 +uvicorn[standard]==0.30.1 # via + # feast (setup.py) # fastapi - # fastapi-cli uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit -watchfiles==0.21.0 + # via + # feast (setup.py) + # pre-commit +watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit -webcolors==1.13 +webcolors==24.6.0 # via jsonschema webencodings==0.5.1 # via @@ -984,5 +1105,5 @@ xmltodict==0.13.0 # via moto yarl==1.9.4 # via aiohttp -zipp==3.18.1 +zipp==3.19.2 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 149a96626e..3c833438de 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -67,8 +73,11 @@ importlib-metadata==7.1.0 # dask # typeguard jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -80,13 +89,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -98,20 +110,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -122,6 +143,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -129,6 +151,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -144,11 +167,15 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -156,7 +183,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -181,6 +210,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index e78c1053bf..c6b034e2aa 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -39,13 +39,18 @@ @pytest.mark.integration @pytest.mark.universal_online_stores(only=["postgres"]) +@pytest.mark.parametrize( + "conn_type", + [ConnectionType.singleton, ConnectionType.pool], + ids=lambda v: f"conn_type:{v}", +) def test_connection_pool_online_stores( - environment, universal_data_sources, fake_ingest_data + environment, universal_data_sources, fake_ingest_data, conn_type ): if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": return fs = environment.feature_store - fs.config.online_store.conn_type = ConnectionType.pool + fs.config.online_store.conn_type = conn_type fs.config.online_store.min_conn = 1 fs.config.online_store.max_conn = 10 diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index 24ba9fe42a..c119ae800a 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -149,7 +149,9 @@ def pg_registry(): registry_config = RegistryConfig( registry_type="sql", - path=f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", + # The `path` must include `+psycopg` in order for `sqlalchemy.create_engine()` + # to understand that we are using psycopg3. + path=f"postgresql+psycopg://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) diff --git a/setup.py b/setup.py index cffd91a0c5..958e93799d 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ TRINO_REQUIRED = ["trino>=0.305.0,<0.400.0", "regex"] POSTGRES_REQUIRED = [ - "psycopg2-binary>=2.8.3,<3", + "psycopg[binary,pool]>=3.0.0,<4", ] MYSQL_REQUIRED = ["pymysql", "types-PyMySQL"] From 7072fd0e2e1d2f4d9a3e8f02d04ae042b3d9c0d4 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 2 Jul 2024 01:05:44 +0400 Subject: [PATCH 06/33] feat: Move get_online_features to OnlineStore interface (#4319) * move get_online_features to OnlineStore interface Signed-off-by: tokoko * fix pydantic warnings Signed-off-by: tokoko * run ruff format Signed-off-by: tokoko --------- Signed-off-by: tokoko --- sdk/python/feast/feature_store.py | 185 +----------------- .../kubernetes/k8s_materialization_engine.py | 4 +- .../feast/infra/online_stores/online_store.py | 182 ++++++++++++++++- .../feast/infra/passthrough_provider.py | 46 ++++- sdk/python/feast/infra/provider.py | 34 +++- sdk/python/tests/foo_provider.py | 32 ++- .../universal/data_sources/file.py | 2 +- 7 files changed, 303 insertions(+), 182 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index b7e4ef619f..6476af5ac8 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1559,75 +1559,16 @@ def get_online_features( ... ) >>> online_response_dict = online_response.to_dict() """ - if isinstance(entity_rows, list): - columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} - for entity_row in entity_rows: - for key, value in entity_row.items(): - try: - columnar[key].append(value) - except KeyError as e: - raise ValueError( - "All entity_rows must have the same keys." - ) from e - - entity_rows = columnar + provider = self._get_provider() - ( - join_key_values, - grouped_refs, - entity_name_to_join_key_map, - requested_on_demand_feature_views, - feature_refs, - requested_result_row_names, - online_features_response, - ) = utils._prepare_entities_to_read_from_online_store( + return provider.get_online_features( + config=self.config, + features=features, + entity_rows=entity_rows, registry=self._registry, project=self.project, - features=features, - entity_values=entity_rows, full_feature_names=full_feature_names, - native_entity_values=True, - ) - - provider = self._get_provider() - for table, requested_features in grouped_refs: - # Get the correct set of entity values with the correct join keys. - table_entity_values, idxs = utils._get_unique_entities( - table, - join_key_values, - entity_name_to_join_key_map, - ) - - # Fetch feature data for the minimum set of Entities. - feature_data = self._read_from_online_store( - table_entity_values, - provider, - requested_features, - table, - ) - - # Populate the result_rows with the Features from the OnlineStore inplace. - utils._populate_response_from_feature_data( - feature_data, - idxs, - online_features_response, - full_feature_names, - requested_features, - table, - ) - - if requested_on_demand_feature_views: - utils._augment_response_with_on_demand_transforms( - online_features_response, - feature_refs, - requested_on_demand_feature_views, - full_feature_names, - ) - - utils._drop_unneeded_columns( - online_features_response, requested_result_row_names ) - return OnlineResponse(online_features_response) async def get_online_features_async( self, @@ -1664,75 +1605,16 @@ async def get_online_features_async( Raises: Exception: No entity with the specified name exists. """ - if isinstance(entity_rows, list): - columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} - for entity_row in entity_rows: - for key, value in entity_row.items(): - try: - columnar[key].append(value) - except KeyError as e: - raise ValueError( - "All entity_rows must have the same keys." - ) from e - - entity_rows = columnar + provider = self._get_provider() - ( - join_key_values, - grouped_refs, - entity_name_to_join_key_map, - requested_on_demand_feature_views, - feature_refs, - requested_result_row_names, - online_features_response, - ) = utils._prepare_entities_to_read_from_online_store( + return await provider.get_online_features_async( + config=self.config, + features=features, + entity_rows=entity_rows, registry=self._registry, project=self.project, - features=features, - entity_values=entity_rows, full_feature_names=full_feature_names, - native_entity_values=True, - ) - - provider = self._get_provider() - for table, requested_features in grouped_refs: - # Get the correct set of entity values with the correct join keys. - table_entity_values, idxs = utils._get_unique_entities( - table, - join_key_values, - entity_name_to_join_key_map, - ) - - # Fetch feature data for the minimum set of Entities. - feature_data = await self._read_from_online_store_async( - table_entity_values, - provider, - requested_features, - table, - ) - - # Populate the result_rows with the Features from the OnlineStore inplace. - utils._populate_response_from_feature_data( - feature_data, - idxs, - online_features_response, - full_feature_names, - requested_features, - table, - ) - - if requested_on_demand_feature_views: - utils._augment_response_with_on_demand_transforms( - online_features_response, - feature_refs, - requested_on_demand_feature_views, - full_feature_names, - ) - - utils._drop_unneeded_columns( - online_features_response, requested_result_row_names ) - return OnlineResponse(online_features_response) def retrieve_online_documents( self, @@ -1806,53 +1688,6 @@ def retrieve_online_documents( ) return OnlineResponse(online_features_response) - def _read_from_online_store( - self, - entity_rows: Iterable[Mapping[str, Value]], - provider: Provider, - requested_features: List[str], - table: FeatureView, - ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: - """Read and process data from the OnlineStore for a given FeatureView. - - This method guarantees that the order of the data in each element of the - List returned is the same as the order of `requested_features`. - - This method assumes that `provider.online_read` returns data for each - combination of Entities in `entity_rows` in the same order as they - are provided. - """ - entity_key_protos = utils._get_entity_key_protos(entity_rows) - - # Fetch data for Entities. - read_rows = provider.online_read( - config=self.config, - table=table, - entity_keys=entity_key_protos, - requested_features=requested_features, - ) - - return utils._convert_rows_to_protobuf(requested_features, read_rows) - - async def _read_from_online_store_async( - self, - entity_rows: Iterable[Mapping[str, Value]], - provider: Provider, - requested_features: List[str], - table: FeatureView, - ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: - entity_key_protos = utils._get_entity_key_protos(entity_rows) - - # Fetch data for Entities. - read_rows = await provider.online_read_async( - config=self.config, - table=table, - entity_keys=entity_key_protos, - requested_features=requested_features, - ) - - return utils._convert_rows_to_protobuf(requested_features, read_rows) - def _retrieve_from_online_store( self, provider: Provider, diff --git a/sdk/python/feast/infra/materialization/kubernetes/k8s_materialization_engine.py b/sdk/python/feast/infra/materialization/kubernetes/k8s_materialization_engine.py index 2e7129b037..510b6b4e4c 100644 --- a/sdk/python/feast/infra/materialization/kubernetes/k8s_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/kubernetes/k8s_materialization_engine.py @@ -306,7 +306,9 @@ def _create_kubernetes_job(self, job_id, paths, feature_view): def _create_configuration_map(self, job_id, paths, feature_view, namespace): """Create a Kubernetes configmap for this job""" - feature_store_configuration = yaml.dump(self.repo_config.dict(by_alias=True)) + feature_store_configuration = yaml.dump( + self.repo_config.model_dump(by_alias=True) + ) materialization_config = yaml.dump( {"paths": paths, "feature_view": feature_view.name} diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 05983a494c..9cf2ef95f6 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -14,13 +14,17 @@ from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union -from feast import Entity +from feast import Entity, utils +from feast.feature_service import FeatureService from feast.feature_view import FeatureView from feast.infra.infra_object import InfraObject +from feast.infra.registry.base_registry import BaseRegistry +from feast.online_response import OnlineResponse from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import RepeatedValue from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RepoConfig @@ -105,6 +109,180 @@ async def online_read_async( f"Online store {self.__class__.__name__} does not support online read async" ) + def get_online_features( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + if isinstance(entity_rows, list): + columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} + for entity_row in entity_rows: + for key, value in entity_row.items(): + try: + columnar[key].append(value) + except KeyError as e: + raise ValueError( + "All entity_rows must have the same keys." + ) from e + + entity_rows = columnar + + ( + join_key_values, + grouped_refs, + entity_name_to_join_key_map, + requested_on_demand_feature_views, + feature_refs, + requested_result_row_names, + online_features_response, + ) = utils._prepare_entities_to_read_from_online_store( + registry=registry, + project=project, + features=features, + entity_values=entity_rows, + full_feature_names=full_feature_names, + native_entity_values=True, + ) + + for table, requested_features in grouped_refs: + # Get the correct set of entity values with the correct join keys. + table_entity_values, idxs = utils._get_unique_entities( + table, + join_key_values, + entity_name_to_join_key_map, + ) + + entity_key_protos = utils._get_entity_key_protos(table_entity_values) + + # Fetch data for Entities. + read_rows = self.online_read( + config=config, + table=table, + entity_keys=entity_key_protos, + requested_features=requested_features, + ) + + feature_data = utils._convert_rows_to_protobuf( + requested_features, read_rows + ) + + # Populate the result_rows with the Features from the OnlineStore inplace. + utils._populate_response_from_feature_data( + feature_data, + idxs, + online_features_response, + full_feature_names, + requested_features, + table, + ) + + if requested_on_demand_feature_views: + utils._augment_response_with_on_demand_transforms( + online_features_response, + feature_refs, + requested_on_demand_feature_views, + full_feature_names, + ) + + utils._drop_unneeded_columns( + online_features_response, requested_result_row_names + ) + return OnlineResponse(online_features_response) + + async def get_online_features_async( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + if isinstance(entity_rows, list): + columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} + for entity_row in entity_rows: + for key, value in entity_row.items(): + try: + columnar[key].append(value) + except KeyError as e: + raise ValueError( + "All entity_rows must have the same keys." + ) from e + + entity_rows = columnar + + ( + join_key_values, + grouped_refs, + entity_name_to_join_key_map, + requested_on_demand_feature_views, + feature_refs, + requested_result_row_names, + online_features_response, + ) = utils._prepare_entities_to_read_from_online_store( + registry=registry, + project=project, + features=features, + entity_values=entity_rows, + full_feature_names=full_feature_names, + native_entity_values=True, + ) + + for table, requested_features in grouped_refs: + # Get the correct set of entity values with the correct join keys. + table_entity_values, idxs = utils._get_unique_entities( + table, + join_key_values, + entity_name_to_join_key_map, + ) + + entity_key_protos = utils._get_entity_key_protos(table_entity_values) + + # Fetch data for Entities. + read_rows = await self.online_read_async( + config=config, + table=table, + entity_keys=entity_key_protos, + requested_features=requested_features, + ) + + feature_data = utils._convert_rows_to_protobuf( + requested_features, read_rows + ) + + # Populate the result_rows with the Features from the OnlineStore inplace. + utils._populate_response_from_feature_data( + feature_data, + idxs, + online_features_response, + full_feature_names, + requested_features, + table, + ) + + if requested_on_demand_feature_views: + utils._augment_response_with_on_demand_transforms( + online_features_response, + feature_refs, + requested_on_demand_feature_views, + full_feature_names, + ) + + utils._drop_unneeded_columns( + online_features_response, requested_result_row_names + ) + return OnlineResponse(online_features_response) + @abstractmethod def update( self, diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index bad6f86cc6..c3c3048a89 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union import pandas as pd import pyarrow as pa @@ -23,8 +23,10 @@ from feast.infra.online_stores.helpers import get_online_store_from_config from feast.infra.provider import Provider from feast.infra.registry.base_registry import BaseRegistry +from feast.online_response import OnlineResponse from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import RepeatedValue from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import BATCH_ENGINE_CLASS_FOR_TYPE, RepoConfig from feast.saved_dataset import SavedDataset @@ -193,6 +195,48 @@ def online_read( ) return result + def get_online_features( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + return self.online_store.get_online_features( + config=config, + features=features, + entity_rows=entity_rows, + registry=registry, + project=project, + full_feature_names=full_feature_names, + ) + + async def get_online_features_async( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + return await self.online_store.get_online_features_async( + config=config, + features=features, + entity_rows=entity_rows, + registry=registry, + project=project, + full_feature_names=full_feature_names, + ) + async def online_read_async( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 75afd6bba8..9940af1d02 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union import pandas as pd import pyarrow @@ -15,8 +15,10 @@ from feast.infra.infra_object import Infra from feast.infra.offline_stores.offline_store import RetrievalJob from feast.infra.registry.base_registry import BaseRegistry +from feast.online_response import OnlineResponse from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import RepeatedValue from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RepoConfig from feast.saved_dataset import SavedDataset @@ -230,6 +232,36 @@ def online_read( """ pass + @abstractmethod + def get_online_features( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + pass + + @abstractmethod + async def get_online_features_async( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + pass + @abstractmethod async def online_read_async( self, diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index bd1e247a7b..8e8f54db24 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -1,6 +1,6 @@ from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union import pandas import pyarrow @@ -11,7 +11,9 @@ from feast.infra.offline_stores.offline_store import RetrievalJob from feast.infra.provider import Provider from feast.infra.registry.base_registry import BaseRegistry +from feast.online_response import OnlineResponse from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import RepeatedValue from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.saved_dataset import SavedDataset @@ -138,3 +140,31 @@ def validate_data_source( data_source: DataSource, ): pass + + def get_online_features( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + pass + + async def get_online_features_async( + self, + config: RepoConfig, + features: Union[List[str], FeatureService], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValue]], + ], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> OnlineResponse: + pass diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index f7ab55d868..4a4a7360d8 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -381,7 +381,7 @@ def setup(self, registry: RegistryConfig): repo_path = Path(tempfile.mkdtemp()) with open(repo_path / "feature_store.yaml", "w") as outfile: - yaml.dump(config.dict(by_alias=True), outfile) + yaml.dump(config.model_dump(by_alias=True), outfile) repo_path = str(repo_path.resolve()) self.server_port = free_port() From 398ea3b86c83605963124404ff4baa95162dc1f4 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 1 Jul 2024 17:06:27 -0400 Subject: [PATCH 07/33] fix: Fix SQLite import issue (#4294) adding try and except block for import Co-authored-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 41af14aaf1..9896b766d4 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import itertools +import logging import os import sqlite3 import struct @@ -20,7 +21,6 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union -import sqlite_vec from google.protobuf.internal.containers import RepeatedScalarFieldContainer from pydantic import StrictStr @@ -84,7 +84,9 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - if sys.version_info[0:2] == (3, 10): + if sys.version_info[0:2] == (3, 10) and config.online_store.vec_enabled: + import sqlite_vec # noqa: F401 + self._conn.enable_load_extension(True) # type: ignore sqlite_vec.load(self._conn) @@ -410,6 +412,10 @@ def retrieve_online_documents( def _initialize_conn(db_path: str): + try: + import sqlite_vec # noqa: F401 + except ModuleNotFoundError: + logging.warning("Cannot use sqlite_vec for vector search") Path(db_path).parent.mkdir(exist_ok=True) return sqlite3.connect( db_path, @@ -482,8 +488,13 @@ def from_proto(sqlite_table_proto: SqliteTableProto) -> Any: def update(self): if sys.version_info[0:2] == (3, 10): - self.conn.enable_load_extension(True) - sqlite_vec.load(self.conn) + try: + import sqlite_vec # noqa: F401 + + self.conn.enable_load_extension(True) + sqlite_vec.load(self.conn) + except ModuleNotFoundError: + logging.warning("Cannot use sqlite_vec for vector search") self.conn.execute( f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) From 98ff63cd389207998b3452ec46e5a2f0fc70485c Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Mon, 1 Jul 2024 22:54:28 -0400 Subject: [PATCH 08/33] fix: Using one single function call for utcnow(). (#4307) Signed-off-by: Shuchu Han --- sdk/python/feast/feature_store.py | 9 +++--- .../feast/infra/offline_stores/bigquery.py | 4 +-- .../trino_offline_store/trino_queries.py | 6 ++-- .../feast/infra/online_stores/datastore.py | 6 ++-- .../feast/infra/registry/caching_registry.py | 9 +++--- .../contrib/azure/azure_registry_store.py | 4 +-- sdk/python/feast/infra/registry/file.py | 4 +-- sdk/python/feast/infra/registry/gcs.py | 4 +-- sdk/python/feast/infra/registry/registry.py | 21 ++++++------ sdk/python/feast/infra/registry/s3.py | 4 +-- sdk/python/feast/infra/registry/snowflake.py | 22 ++++++------- sdk/python/feast/infra/registry/sql.py | 9 +++--- sdk/python/feast/on_demand_feature_view.py | 6 ++-- sdk/python/feast/utils.py | 4 +++ sdk/python/tests/conftest.py | 11 ++++--- sdk/python/tests/data/data_creator.py | 15 +++++---- sdk/python/tests/doctest/test_all.py | 7 ++-- .../feature_repos/repo_configuration.py | 3 +- .../materialization/test_snowflake.py | 5 +-- .../offline_store/test_offline_write.py | 9 +++--- .../test_push_features_to_offline_store.py | 5 ++- .../test_universal_historical_retrieval.py | 13 ++++---- .../offline_store/test_validation.py | 5 ++- .../test_push_features_to_online_store.py | 7 ++-- .../test_python_feature_server.py | 10 +++--- .../online_store/test_remote_online_store.py | 4 +-- .../online_store/test_universal_online.py | 13 ++++---- .../test_universal_odfv_feature_inference.py | 7 ++-- .../registration/test_universal_registry.py | 7 ++-- .../registration/test_universal_types.py | 3 +- sdk/python/tests/unit/cli/test_cli_chdir.py | 5 +-- .../tests/unit/local_feast_tests/test_init.py | 5 +-- .../online_store/test_online_retrieval.py | 32 +++++++++---------- sdk/python/tests/unit/test_datetime.py | 6 ++++ sdk/python/tests/unit/test_feature_views.py | 10 +++--- .../tests/unit/test_stream_feature_view.py | 10 +++--- .../tests/utils/basic_read_write_test.py | 7 ++-- .../tests/utils/dynamo_table_creator.py | 5 ++- sdk/python/tests/utils/e2e_test_validation.py | 3 +- .../tests/utils/online_write_benchmark.py | 6 ++-- sdk/python/tests/utils/test_log_creator.py | 8 ++--- 41 files changed, 176 insertions(+), 157 deletions(-) create mode 100644 sdk/python/tests/unit/test_datetime.py diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 6476af5ac8..9600732e17 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -86,6 +86,7 @@ from feast.repo_contents import RepoContents from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.stream_feature_view import StreamFeatureView +from feast.utils import _utc_now from feast.version import get_version warnings.simplefilter("once", DeprecationWarning) @@ -1246,7 +1247,7 @@ def materialize_incremental( >>> from feast import FeatureStore, RepoConfig >>> from datetime import datetime, timedelta >>> fs = FeatureStore(repo_path="project/feature_repo") - >>> fs.materialize_incremental(end_date=datetime.utcnow() - timedelta(minutes=5)) + >>> fs.materialize_incremental(end_date=_utc_now() - timedelta(minutes=5)) Materializing... ... @@ -1270,7 +1271,7 @@ def materialize_incremental( f" either a ttl to be set or for materialize() to have been run at least once." ) elif feature_view.ttl.total_seconds() > 0: - start_date = datetime.utcnow() - feature_view.ttl + start_date = _utc_now() - feature_view.ttl else: # TODO(felixwang9817): Find the earliest timestamp for this specific feature # view from the offline store, and set the start date to that timestamp. @@ -1278,7 +1279,7 @@ def materialize_incremental( f"Since the ttl is 0 for feature view {Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}, " "the start date will be set to 1 year before the current time." ) - start_date = datetime.utcnow() - timedelta(weeks=52) + start_date = _utc_now() - timedelta(weeks=52) provider = self._get_provider() print( f"{Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}" @@ -1335,7 +1336,7 @@ def materialize( >>> from datetime import datetime, timedelta >>> fs = FeatureStore(repo_path="project/feature_repo") >>> fs.materialize( - ... start_date=datetime.utcnow() - timedelta(hours=3), end_date=datetime.utcnow() - timedelta(minutes=10) + ... start_date=_utc_now() - timedelta(hours=3), end_date=_utc_now() - timedelta(minutes=10) ... ) Materializing... diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 36334b606d..3e4a0f1b99 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -45,7 +45,7 @@ from feast.on_demand_feature_view import OnDemandFeatureView from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage -from feast.utils import get_user_agent +from feast.utils import _utc_now, get_user_agent from .bigquery_source import ( BigQueryLoggingDestination, @@ -701,7 +701,7 @@ def _upload_entity_df( # Ensure that the table expires after some time table = client.get_table(table=table_name) - table.expires = datetime.utcnow() + timedelta(minutes=30) + table.expires = _utc_now() + timedelta(minutes=30) client.update_table(table, ["expires"]) return table diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py index 50472407bc..3a26583af2 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py @@ -1,6 +1,5 @@ from __future__ import annotations -import datetime import signal from dataclasses import dataclass from enum import Enum @@ -16,6 +15,7 @@ from feast.infra.offline_stores.contrib.trino_offline_store.trino_type_map import ( trino_to_pa_value_type, ) +from feast.utils import _utc_now class QueryStatus(Enum): @@ -97,12 +97,12 @@ def __init__(self, query_text: str, cursor: Cursor): def execute(self) -> Results: try: self.status = QueryStatus.RUNNING - start_time = datetime.datetime.utcnow() + start_time = _utc_now() self._cursor.execute(operation=self.query_text) rows = self._cursor.fetchall() - end_time = datetime.datetime.utcnow() + end_time = _utc_now() self.execution_time = end_time - start_time self.status = QueryStatus.COMPLETED diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index b33767cea5..9ae10792f5 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -44,7 +44,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.utils import get_user_agent +from feast.utils import _utc_now, get_user_agent LOGGER = logging.getLogger(__name__) @@ -122,7 +122,7 @@ def update( entity = datastore.Entity( key=key, exclude_from_indexes=("created_ts", "event_ts", "values") ) - entity.update({"created_ts": datetime.utcnow()}) + entity.update({"created_ts": _utc_now()}) client.put(entity) for table in tables_to_delete: @@ -457,7 +457,7 @@ def update(self): entity = datastore.Entity( key=key, exclude_from_indexes=("created_ts", "event_ts", "values") ) - entity.update({"created_ts": datetime.utcnow()}) + entity.update({"created_ts": _utc_now()}) client.put(entity) def teardown(self): diff --git a/sdk/python/feast/infra/registry/caching_registry.py b/sdk/python/feast/infra/registry/caching_registry.py index 6336dd7fee..f7eab7d70a 100644 --- a/sdk/python/feast/infra/registry/caching_registry.py +++ b/sdk/python/feast/infra/registry/caching_registry.py @@ -1,6 +1,6 @@ import logging from abc import abstractmethod -from datetime import datetime, timedelta +from datetime import timedelta from threading import Lock from typing import List, Optional @@ -15,6 +15,7 @@ from feast.project_metadata import ProjectMetadata from feast.saved_dataset import SavedDataset, ValidationReference from feast.stream_feature_view import StreamFeatureView +from feast.utils import _utc_now logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ def __init__( ): self.cached_registry_proto = self.proto() proto_registry_utils.init_project_metadata(self.cached_registry_proto, project) - self.cached_registry_proto_created = datetime.utcnow() + self.cached_registry_proto_created = _utc_now() self._refresh_lock = Lock() self.cached_registry_proto_ttl = timedelta( seconds=cache_ttl_seconds if cache_ttl_seconds is not None else 0 @@ -318,7 +319,7 @@ def refresh(self, project: Optional[str] = None): self.cached_registry_proto, project ) self.cached_registry_proto = self.proto() - self.cached_registry_proto_created = datetime.utcnow() + self.cached_registry_proto_created = _utc_now() def _refresh_cached_registry_if_necessary(self): with self._refresh_lock: @@ -329,7 +330,7 @@ def _refresh_cached_registry_if_necessary(self): self.cached_registry_proto_ttl.total_seconds() > 0 # 0 ttl means infinity and ( - datetime.utcnow() + _utc_now() > ( self.cached_registry_proto_created + self.cached_registry_proto_ttl diff --git a/sdk/python/feast/infra/registry/contrib/azure/azure_registry_store.py b/sdk/python/feast/infra/registry/contrib/azure/azure_registry_store.py index 9c00170b0f..f9317bf7a4 100644 --- a/sdk/python/feast/infra/registry/contrib/azure/azure_registry_store.py +++ b/sdk/python/feast/infra/registry/contrib/azure/azure_registry_store.py @@ -3,7 +3,6 @@ import os import uuid -from datetime import datetime from pathlib import Path from tempfile import TemporaryFile from urllib.parse import urlparse @@ -11,6 +10,7 @@ from feast.infra.registry.registry import RegistryConfig from feast.infra.registry.registry_store import RegistryStore from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto +from feast.utils import _utc_now REGISTRY_SCHEMA_VERSION = "1" @@ -89,7 +89,7 @@ def teardown(self): def _write_registry(self, registry_proto: RegistryProto): registry_proto.version_id = str(uuid.uuid4()) - registry_proto.last_updated.FromDatetime(datetime.utcnow()) + registry_proto.last_updated.FromDatetime(_utc_now()) file_obj = TemporaryFile() file_obj.write(registry_proto.SerializeToString()) diff --git a/sdk/python/feast/infra/registry/file.py b/sdk/python/feast/infra/registry/file.py index 7117a0d2c6..ae783bf82c 100644 --- a/sdk/python/feast/infra/registry/file.py +++ b/sdk/python/feast/infra/registry/file.py @@ -1,10 +1,10 @@ import uuid -from datetime import datetime from pathlib import Path from feast.infra.registry.registry_store import RegistryStore from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.repo_config import RegistryConfig +from feast.utils import _utc_now class FileRegistryStore(RegistryStore): @@ -37,7 +37,7 @@ def teardown(self): def _write_registry(self, registry_proto: RegistryProto): registry_proto.version_id = str(uuid.uuid4()) - registry_proto.last_updated.FromDatetime(datetime.utcnow()) + registry_proto.last_updated.FromDatetime(_utc_now()) file_dir = self._filepath.parent file_dir.mkdir(exist_ok=True) with open(self._filepath, mode="wb", buffering=0) as f: diff --git a/sdk/python/feast/infra/registry/gcs.py b/sdk/python/feast/infra/registry/gcs.py index 7e4b7104cf..72498ad054 100644 --- a/sdk/python/feast/infra/registry/gcs.py +++ b/sdk/python/feast/infra/registry/gcs.py @@ -1,5 +1,4 @@ import uuid -from datetime import datetime from pathlib import Path from tempfile import TemporaryFile from urllib.parse import urlparse @@ -7,6 +6,7 @@ from feast.infra.registry.registry_store import RegistryStore from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.repo_config import RegistryConfig +from feast.utils import _utc_now class GCSRegistryStore(RegistryStore): @@ -62,7 +62,7 @@ def teardown(self): def _write_registry(self, registry_proto: RegistryProto): registry_proto.version_id = str(uuid.uuid4()) - registry_proto.last_updated.FromDatetime(datetime.utcnow()) + registry_proto.last_updated.FromDatetime(_utc_now()) # we have already checked the bucket exists so no need to do it again gs_bucket = self.gcs_client.get_bucket(self._bucket) blob = gs_bucket.blob(self._blob) diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index 4d6bff4cc7..fe44e6253a 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -47,6 +47,7 @@ from feast.repo_contents import RepoContents from feast.saved_dataset import SavedDataset, ValidationReference from feast.stream_feature_view import StreamFeatureView +from feast.utils import _utc_now REGISTRY_SCHEMA_VERSION = "1" @@ -217,7 +218,7 @@ def clone(self) -> "Registry": if self.cached_registry_proto else RegistryProto() ) - new_registry.cached_registry_proto_created = datetime.utcnow() + new_registry.cached_registry_proto_created = _utc_now() new_registry._registry_store = NoopRegistryStore() return new_registry @@ -248,7 +249,7 @@ def get_infra(self, project: str, allow_cache: bool = False) -> Infra: def apply_entity(self, entity: Entity, project: str, commit: bool = True): entity.is_valid() - now = datetime.utcnow() + now = _utc_now() if not entity.created_timestamp: entity.created_timestamp = now entity.last_updated_timestamp = now @@ -334,7 +335,7 @@ def delete_data_source(self, name: str, project: str, commit: bool = True): def apply_feature_service( self, feature_service: FeatureService, project: str, commit: bool = True ): - now = datetime.utcnow() + now = _utc_now() if not feature_service.created_timestamp: feature_service.created_timestamp = now feature_service.last_updated_timestamp = now @@ -390,7 +391,7 @@ def apply_feature_view( ): feature_view.ensure_valid() - now = datetime.utcnow() + now = _utc_now() if not feature_view.created_timestamp: feature_view.created_timestamp = now feature_view.last_updated_timestamp = now @@ -517,7 +518,7 @@ def apply_materialization( existing_feature_view.materialization_intervals.append( (start_date, end_date) ) - existing_feature_view.last_updated_timestamp = datetime.utcnow() + existing_feature_view.last_updated_timestamp = _utc_now() feature_view_proto = existing_feature_view.to_proto() feature_view_proto.spec.project = project del self.cached_registry_proto.feature_views[idx] @@ -539,7 +540,7 @@ def apply_materialization( existing_stream_feature_view.materialization_intervals.append( (start_date, end_date) ) - existing_stream_feature_view.last_updated_timestamp = datetime.utcnow() + existing_stream_feature_view.last_updated_timestamp = _utc_now() stream_feature_view_proto = existing_stream_feature_view.to_proto() stream_feature_view_proto.spec.project = project del self.cached_registry_proto.stream_feature_views[idx] @@ -664,7 +665,7 @@ def apply_saved_dataset( project: str, commit: bool = True, ): - now = datetime.utcnow() + now = _utc_now() if not saved_dataset.created_timestamp: saved_dataset.created_timestamp = now saved_dataset.last_updated_timestamp = now @@ -812,7 +813,7 @@ def _prepare_registry_for_changes(self, project: str): registry_proto = RegistryProto() registry_proto.registry_schema_version = REGISTRY_SCHEMA_VERSION self.cached_registry_proto = registry_proto - self.cached_registry_proto_created = datetime.utcnow() + self.cached_registry_proto_created = _utc_now() # Initialize project metadata if needed assert self.cached_registry_proto @@ -848,7 +849,7 @@ def _get_registry_proto( self.cached_registry_proto_ttl.total_seconds() > 0 # 0 ttl means infinity and ( - datetime.utcnow() + _utc_now() > ( self.cached_registry_proto_created + self.cached_registry_proto_ttl @@ -871,7 +872,7 @@ def _get_registry_proto( logger.info("Registry cache expired, so refreshing") registry_proto = self._registry_store.get_registry_proto() self.cached_registry_proto = registry_proto - self.cached_registry_proto_created = datetime.utcnow() + self.cached_registry_proto_created = _utc_now() if not project: return registry_proto diff --git a/sdk/python/feast/infra/registry/s3.py b/sdk/python/feast/infra/registry/s3.py index cbae3af11c..8aac4d52ee 100644 --- a/sdk/python/feast/infra/registry/s3.py +++ b/sdk/python/feast/infra/registry/s3.py @@ -1,6 +1,5 @@ import os import uuid -from datetime import datetime from pathlib import Path from tempfile import TemporaryFile from urllib.parse import urlparse @@ -9,6 +8,7 @@ from feast.infra.registry.registry_store import RegistryStore from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.repo_config import RegistryConfig +from feast.utils import _utc_now try: import boto3 @@ -70,7 +70,7 @@ def teardown(self): def _write_registry(self, registry_proto: RegistryProto): registry_proto.version_id = str(uuid.uuid4()) - registry_proto.last_updated.FromDatetime(datetime.utcnow()) + registry_proto.last_updated.FromDatetime(_utc_now()) # we have already checked the bucket exists so no need to do it again file_obj = TemporaryFile() file_obj.write(registry_proto.SerializeToString()) diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index d7ab67e7d0..f2bc09e7e4 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -10,7 +10,6 @@ from pydantic import ConfigDict, Field, StrictStr import feast -from feast import utils from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -54,6 +53,7 @@ from feast.repo_config import RegistryConfig from feast.saved_dataset import SavedDataset, ValidationReference from feast.stream_feature_view import StreamFeatureView +from feast.utils import _utc_now, has_all_tags logger = logging.getLogger(__name__) @@ -126,16 +126,15 @@ def __init__( with GetSnowflakeConnection(self.registry_config) as conn: sql_function_file = f"{os.path.dirname(feast.__file__)}/infra/utils/snowflake/registry/snowflake_table_creation.sql" with open(sql_function_file, "r") as file: - sqlFile = file.read() - - sqlCommands = sqlFile.split(";") - for command in sqlCommands: + sql_file = file.read() + sql_cmds = sql_file.split(";") + for command in sql_cmds: query = command.replace("REGISTRY_PATH", f"{self.registry_path}") execute_snowflake_statement(conn, query) self.cached_registry_proto = self.proto() proto_registry_utils.init_project_metadata(self.cached_registry_proto, project) - self.cached_registry_proto_created = datetime.utcnow() + self.cached_registry_proto_created = _utc_now() self._refresh_lock = Lock() self.cached_registry_proto_ttl = timedelta( seconds=registry_config.cache_ttl_seconds @@ -154,7 +153,7 @@ def refresh(self, project: Optional[str] = None): self.cached_registry_proto, project ) self.cached_registry_proto = self.proto() - self.cached_registry_proto_created = datetime.utcnow() + self.cached_registry_proto_created = _utc_now() def _refresh_cached_registry_if_necessary(self): with self._refresh_lock: @@ -165,7 +164,7 @@ def _refresh_cached_registry_if_necessary(self): self.cached_registry_proto_ttl.total_seconds() > 0 # 0 ttl means infinity and ( - datetime.utcnow() + _utc_now() > ( self.cached_registry_proto_created + self.cached_registry_proto_ttl @@ -182,7 +181,6 @@ def teardown(self): sql_function_file = f"{os.path.dirname(feast.__file__)}/infra/utils/snowflake/registry/snowflake_table_deletion.sql" with open(sql_function_file, "r") as file: sqlFile = file.read() - sqlCommands = sqlFile.split(";") for command in sqlCommands: query = command.replace("REGISTRY_PATH", f"{self.registry_path}") @@ -281,7 +279,7 @@ def _apply_object( name = name or (obj.name if hasattr(obj, "name") else None) assert name, f"name needs to be provided for {obj}" - update_datetime = datetime.utcnow() + update_datetime = _utc_now() if hasattr(obj, "last_updated_timestamp"): obj.last_updated_timestamp = update_datetime @@ -416,7 +414,7 @@ def _delete_object( if cursor.rowcount < 1 and not_found_exception: # type: ignore raise not_found_exception(name, project) - self._set_last_updated_metadata(datetime.utcnow(), project) + self._set_last_updated_metadata(_utc_now(), project) return cursor.rowcount @@ -787,7 +785,7 @@ def _list_objects( obj = python_class.from_proto( proto_class.FromString(row[1][proto_field_name]) ) - if utils.has_all_tags(obj.tags, tags): + if has_all_tags(obj.tags, tags): objects.append(obj) return objects return [] diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 239898677c..6ef08989b7 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -60,6 +60,7 @@ from feast.repo_config import RegistryConfig from feast.saved_dataset import SavedDataset, ValidationReference from feast.stream_feature_view import StreamFeatureView +from feast.utils import _utc_now metadata = MetaData() @@ -591,7 +592,7 @@ def apply_user_metadata( table.c.project_id == project, ) row = conn.execute(stmt).first() - update_datetime = datetime.utcnow() + update_datetime = _utc_now() update_time = int(update_datetime.timestamp()) if row: values = { @@ -703,7 +704,7 @@ def _apply_object( assert name, f"name needs to be provided for {obj}" with self.engine.begin() as conn: - update_datetime = datetime.utcnow() + update_datetime = _utc_now() update_time = int(update_datetime.timestamp()) stmt = select(table).where( getattr(table.c, id_field_name) == name, table.c.project_id == project @@ -770,7 +771,7 @@ def _apply_object( def _maybe_init_project_metadata(self, project): # Initialize project metadata if needed with self.engine.begin() as conn: - update_datetime = datetime.utcnow() + update_datetime = _utc_now() update_time = int(update_datetime.timestamp()) stmt = select(feast_metadata).where( feast_metadata.c.metadata_key == FeastMetadataKeys.PROJECT_UUID.value, @@ -803,7 +804,7 @@ def _delete_object( rows = conn.execute(stmt) if rows.rowcount < 1 and not_found_exception: raise not_found_exception(name, project) - self._set_last_updated_metadata(datetime.utcnow(), project) + self._set_last_updated_metadata(_utc_now(), project) return rows.rowcount diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 839ce4d64c..586f5d1bac 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -2,7 +2,6 @@ import functools import inspect import warnings -from datetime import datetime from types import FunctionType from typing import Any, Optional, Union @@ -34,6 +33,7 @@ from feast.transformation.pandas_transformation import PandasTransformation from feast.transformation.python_transformation import PythonTransformation from feast.transformation.substrait_transformation import SubstraitTransformation +from feast.utils import _utc_now from feast.value_type import ValueType warnings.simplefilter("once", DeprecationWarning) @@ -549,7 +549,7 @@ def _construct_random_input(self) -> dict[str, list[Any]]: ValueType.DOUBLE: [1.0], ValueType.FLOAT: [1.0], ValueType.BOOL: [True], - ValueType.UNIX_TIMESTAMP: [datetime.utcnow()], + ValueType.UNIX_TIMESTAMP: [_utc_now()], ValueType.BYTES_LIST: [[str.encode("hello world")]], ValueType.STRING_LIST: [["hello world"]], ValueType.INT32_LIST: [[1]], @@ -557,7 +557,7 @@ def _construct_random_input(self) -> dict[str, list[Any]]: ValueType.DOUBLE_LIST: [[1.0]], ValueType.FLOAT_LIST: [[1.0]], ValueType.BOOL_LIST: [[True]], - ValueType.UNIX_TIMESTAMP_LIST: [[datetime.utcnow()]], + ValueType.UNIX_TIMESTAMP_LIST: [[_utc_now()]], } feature_dict = {} diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index a6c893c954..1a1d757fc1 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1052,3 +1052,7 @@ def tags_str_to_dict(tags: str = "") -> dict[str, str]: cast(tuple[str, str], tag.split(":", 1)) for tag in tags_list if ":" in tag ).items() } + + +def _utc_now() -> datetime: + return datetime.utcnow() diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index fb6b7e5608..1fd510d104 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -15,7 +15,7 @@ import multiprocessing import os import random -from datetime import datetime, timedelta +from datetime import timedelta from multiprocessing import Process from sys import platform from typing import Any, Dict, List, Tuple, no_type_check @@ -27,6 +27,7 @@ from feast.data_source import DataSource from feast.feature_store import FeatureStore # noqa: E402 +from feast.utils import _utc_now from feast.wait import wait_retry_backoff # noqa: E402 from tests.data.data_creator import ( # noqa: E402 create_basic_driver_dataset, @@ -133,7 +134,7 @@ def pytest_collection_modifyitems(config, items: List[Item]): @pytest.fixture def simple_dataset_1() -> pd.DataFrame: - now = datetime.utcnow() + now = _utc_now() ts = pd.Timestamp(now).round("ms") data = { "id_join_key": [1, 2, 1, 3, 3], @@ -153,7 +154,7 @@ def simple_dataset_1() -> pd.DataFrame: @pytest.fixture def simple_dataset_2() -> pd.DataFrame: - now = datetime.utcnow() + now = _utc_now() ts = pd.Timestamp(now).round("ms") data = { "id_join_key": ["a", "b", "c", "d", "e"], @@ -391,8 +392,8 @@ def fake_ingest_data(): "conv_rate": [0.5], "acc_rate": [0.6], "avg_daily_trips": [4], - "event_timestamp": [pd.Timestamp(datetime.utcnow()).round("ms")], - "created": [pd.Timestamp(datetime.utcnow()).round("ms")], + "event_timestamp": [pd.Timestamp(_utc_now()).round("ms")], + "created": [pd.Timestamp(_utc_now()).round("ms")], } return pd.DataFrame(data) diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index 1be96f753a..15d09c5a40 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -5,6 +5,7 @@ from pytz import timezone, utc from feast.types import FeastType, Float32, Int32, Int64, String +from feast.utils import _utc_now def create_basic_driver_dataset( @@ -13,7 +14,7 @@ def create_basic_driver_dataset( feature_is_list: bool = False, list_has_empty_list: bool = False, ) -> pd.DataFrame: - now = datetime.utcnow().replace(microsecond=0, second=0, minute=0) + now = _utc_now().replace(microsecond=0, second=0, minute=0) ts = pd.Timestamp(now).round("ms") data = { "driver_id": get_entities_for_feast_type(entity_type), @@ -86,14 +87,14 @@ def create_document_dataset() -> pd.DataFrame: "embedding_float": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "embedding_double": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "ts": [ - pd.Timestamp(datetime.utcnow()).round("ms"), - pd.Timestamp(datetime.utcnow()).round("ms"), - pd.Timestamp(datetime.utcnow()).round("ms"), + pd.Timestamp(_utc_now()).round("ms"), + pd.Timestamp(_utc_now()).round("ms"), + pd.Timestamp(_utc_now()).round("ms"), ], "created_ts": [ - pd.Timestamp(datetime.utcnow()).round("ms"), - pd.Timestamp(datetime.utcnow()).round("ms"), - pd.Timestamp(datetime.utcnow()).round("ms"), + pd.Timestamp(_utc_now()).round("ms"), + pd.Timestamp(_utc_now()).round("ms"), + pd.Timestamp(_utc_now()).round("ms"), ], } return pd.DataFrame(data) diff --git a/sdk/python/tests/doctest/test_all.py b/sdk/python/tests/doctest/test_all.py index 814a7ca798..52348e7da4 100644 --- a/sdk/python/tests/doctest/test_all.py +++ b/sdk/python/tests/doctest/test_all.py @@ -6,13 +6,14 @@ import unittest import feast +from feast.utils import _utc_now FILES_TO_IGNORE = {"app"} def setup_feature_store(): """Prepares the local environment for a FeatureStore docstring test.""" - from datetime import datetime, timedelta + from datetime import timedelta from feast import Entity, FeatureStore, FeatureView, Field, FileSource from feast.repo_operations import init_repo @@ -42,8 +43,8 @@ def setup_feature_store(): ) fs.apply([driver_hourly_stats_view, driver]) fs.materialize( - start_date=datetime.utcnow() - timedelta(hours=3), - end_date=datetime.utcnow() - timedelta(minutes=10), + start_date=_utc_now() - timedelta(hours=3), + end_date=_utc_now() - timedelta(minutes=10), ) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 9e3c02b9c0..48f5070f1e 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -21,6 +21,7 @@ ) from feast.infra.feature_servers.local_process.config import LocalFeatureServerConfig from feast.repo_config import RegistryConfig, RepoConfig +from feast.utils import _utc_now from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, RegistryLocation, @@ -412,7 +413,7 @@ class Environment: fixture_request: Optional[pytest.FixtureRequest] = None def __post_init__(self): - self.end_date = datetime.utcnow().replace(microsecond=0, second=0, minute=0) + self.end_date = _utc_now().replace(microsecond=0, second=0, minute=0) self.start_date: datetime = self.end_date - timedelta(days=3) def setup(self): diff --git a/sdk/python/tests/integration/materialization/test_snowflake.py b/sdk/python/tests/integration/materialization/test_snowflake.py index adb2bd7e7d..f12191363b 100644 --- a/sdk/python/tests/integration/materialization/test_snowflake.py +++ b/sdk/python/tests/integration/materialization/test_snowflake.py @@ -8,6 +8,7 @@ from feast.entity import Entity from feast.feature_view import FeatureView from feast.types import Array, Bool, Bytes, Float64, Int32, Int64, String, UnixTimestamp +from feast.utils import _utc_now from tests.data.data_creator import create_basic_driver_dataset from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, @@ -146,7 +147,7 @@ def test_snowflake_materialization_consistency_internal_with_lists( split_dt = df["ts_1"][4].to_pydatetime() - timedelta(seconds=1) print(f"Split datetime: {split_dt}") - now = datetime.utcnow() + now = _utc_now() full_feature_names = True start_date = (now - timedelta(hours=5)).replace(tzinfo=utc) @@ -231,7 +232,7 @@ def test_snowflake_materialization_entityless_fv(): print(f"Split datetime: {split_dt}") - now = datetime.utcnow() + now = _utc_now() start_date = (now - timedelta(hours=5)).replace(tzinfo=utc) end_date = split_dt diff --git a/sdk/python/tests/integration/offline_store/test_offline_write.py b/sdk/python/tests/integration/offline_store/test_offline_write.py index b8c465946d..63bdc4755a 100644 --- a/sdk/python/tests/integration/offline_store/test_offline_write.py +++ b/sdk/python/tests/integration/offline_store/test_offline_write.py @@ -1,5 +1,5 @@ import random -from datetime import datetime, timedelta +from datetime import timedelta import numpy as np import pandas as pd @@ -7,6 +7,7 @@ from feast import FeatureView, Field from feast.types import Float32, Int32 +from feast.utils import _utc_now from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, ) @@ -23,7 +24,7 @@ def test_reorder_columns(environment, universal_data_sources): driver_fv = feature_views.driver store.apply([driver(), driver_fv]) - now = datetime.utcnow() + now = _utc_now() ts = pd.Timestamp(now).round("ms") # This dataframe has columns in the wrong order. @@ -53,7 +54,7 @@ def test_writing_incorrect_schema_fails(environment, universal_data_sources): driver_fv = feature_views.driver store.apply([driver(), driver_fv]) - now = datetime.utcnow() + now = _utc_now() ts = pd.Timestamp(now).round("ms") expected_df = pd.DataFrame.from_dict( @@ -91,7 +92,7 @@ def test_writing_consecutively_to_offline_store(environment, universal_data_sour ), # This is to make sure all offline store data is out of date since get_historical_features() only searches backwards for a ttl window. ) - now = datetime.utcnow() + now = _utc_now() ts = pd.Timestamp(now, unit="ns") entity_df = pd.DataFrame.from_dict( diff --git a/sdk/python/tests/integration/offline_store/test_push_features_to_offline_store.py b/sdk/python/tests/integration/offline_store/test_push_features_to_offline_store.py index 0b1db9011a..5e3d72e671 100644 --- a/sdk/python/tests/integration/offline_store/test_push_features_to_offline_store.py +++ b/sdk/python/tests/integration/offline_store/test_push_features_to_offline_store.py @@ -1,10 +1,9 @@ -import datetime - import numpy as np import pandas as pd import pytest from feast.data_source import PushMode +from feast.utils import _utc_now from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, ) @@ -20,7 +19,7 @@ def test_push_features_and_read(environment, universal_data_sources): location_fv = feature_views.pushed_locations store.apply([location(), location_fv]) - now = pd.Timestamp(datetime.datetime.utcnow()).round("ms") + now = pd.Timestamp(_utc_now()).round("ms") entity_df = pd.DataFrame.from_dict({"location_id": [1], "event_timestamp": [now]}) before_df = store.get_historical_features( diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index bfb8a56200..ecaa5f40db 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -15,6 +15,7 @@ DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL, ) from feast.types import Float32, Int32 +from feast.utils import _utc_now from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, table_name_from_data_source, @@ -144,11 +145,11 @@ def test_historical_features_main( files = job_from_df.to_remote_storage() assert len(files) # 0 # This test should be way more detailed - start_time = datetime.utcnow() + start_time = _utc_now() actual_df_from_df_entities = job_from_df.to_df() print(f"actual_df_from_df_entities shape: {actual_df_from_df_entities.shape}") - end_time = datetime.utcnow() + end_time = _utc_now() print(str(f"Time to execute job_from_df.to_df() = '{(end_time - start_time)}'\n")) assert sorted(expected_df.columns) == sorted(actual_df_from_df_entities.columns) @@ -303,9 +304,9 @@ def test_historical_features_with_entities_from_query( full_feature_names=full_feature_names, ) - start_time = datetime.utcnow() + start_time = _utc_now() actual_df_from_sql_entities = job_from_sql.to_df() - end_time = datetime.utcnow() + end_time = _utc_now() print(str(f"\nTime to execute job_from_sql.to_df() = '{(end_time - start_time)}'")) event_timestamp = ( @@ -618,11 +619,11 @@ def test_historical_features_containing_backfills(environment): full_feature_names=False, ) - start_time = datetime.utcnow() + start_time = _utc_now() actual_df = offline_job.to_df() print(f"actual_df shape: {actual_df.shape}") - end_time = datetime.utcnow() + end_time = _utc_now() print(str(f"Time to execute job_from_df.to_df() = '{(end_time - start_time)}'\n")) assert sorted(expected_df.columns) == sorted(actual_df.columns) diff --git a/sdk/python/tests/integration/offline_store/test_validation.py b/sdk/python/tests/integration/offline_store/test_validation.py index 1731f823c8..6f0496e8c8 100644 --- a/sdk/python/tests/integration/offline_store/test_validation.py +++ b/sdk/python/tests/integration/offline_store/test_validation.py @@ -16,7 +16,7 @@ LoggingConfig, ) from feast.protos.feast.serving.ServingService_pb2 import FieldStatus -from feast.utils import make_tzaware +from feast.utils import _utc_now, make_tzaware from feast.wait import wait_retry_backoff from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, @@ -316,8 +316,7 @@ def test_e2e_validation_via_cli(environment, universal_data_sources): "avg_passenger_count": [0], "lifetime_trip_count": [0], "event_timestamp": [ - make_tzaware(datetime.datetime.utcnow()) - - datetime.timedelta(hours=1) + make_tzaware(_utc_now()) - datetime.timedelta(hours=1) ], } ) diff --git a/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py b/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py index 42561563f9..98fe3ab1ec 100644 --- a/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py +++ b/sdk/python/tests/integration/online_store/test_push_features_to_online_store.py @@ -1,8 +1,7 @@ -import datetime - import pandas as pd import pytest +from feast.utils import _utc_now from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, ) @@ -21,8 +20,8 @@ def test_push_features_and_read(environment, universal_data_sources): data = { "location_id": [1], "temperature": [4], - "event_timestamp": [pd.Timestamp(datetime.datetime.utcnow()).round("ms")], - "created": [pd.Timestamp(datetime.datetime.utcnow()).round("ms")], + "event_timestamp": [pd.Timestamp(_utc_now()).round("ms")], + "created": [pd.Timestamp(_utc_now()).round("ms")], } df_ingest = pd.DataFrame(data) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 089efd7a56..1010e73178 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -1,5 +1,4 @@ import json -from datetime import datetime from typing import List import pytest @@ -7,6 +6,7 @@ from feast.feast_object import FeastObject from feast.feature_server import get_app +from feast.utils import _utc_now from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, ) @@ -67,8 +67,8 @@ def test_push(python_fs_client): "df": { "location_id": [1], "temperature": [initial_temp * 100], - "event_timestamp": [str(datetime.utcnow())], - "created": [str(datetime.utcnow())], + "event_timestamp": [str(_utc_now())], + "created": [str(_utc_now())], }, } ) @@ -98,8 +98,8 @@ def test_push_source_does_not_exist(python_fs_client): "df": { "location_id": [1], "temperature": [initial_temp * 100], - "event_timestamp": [str(datetime.utcnow())], - "created": [str(datetime.utcnow())], + "event_timestamp": [str(_utc_now())], + "created": [str(_utc_now())], }, } ), diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py index 759a9c7a87..1d5dd0fca0 100644 --- a/sdk/python/tests/integration/online_store/test_remote_online_store.py +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -1,12 +1,12 @@ import os import subprocess import tempfile -from datetime import datetime from textwrap import dedent import pytest from feast.feature_store import FeatureStore +from feast.utils import _utc_now from feast.wait import wait_retry_backoff from tests.utils.cli_repo_creator import CliRunner from tests.utils.http_server import check_port_open, free_port @@ -150,7 +150,7 @@ def _default_store(temp_dir, project_name) -> FeatureStore: fs = FeatureStore(repo_path=repo_path) fs.materialize_incremental( - end_date=datetime.utcnow(), feature_views=["driver_hourly_stats"] + end_date=_utc_now(), feature_views=["driver_hourly_stats"] ) return fs diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index c6b034e2aa..38656b90a9 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -22,6 +22,7 @@ from feast.infra.utils.postgres.postgres_config import ConnectionType from feast.online_response import TIMESTAMP_POSTFIX from feast.types import Float32, Int32, String +from feast.utils import _utc_now from feast.wait import wait_retry_backoff from tests.integration.feature_repos.repo_configuration import ( Environment, @@ -136,9 +137,9 @@ def test_write_to_online_store_event_check(environment): fs = environment.feature_store # write same data points 3 with different timestamps - now = pd.Timestamp(datetime.datetime.utcnow()).round("ms") - hour_ago = pd.Timestamp(datetime.datetime.utcnow() - timedelta(hours=1)).round("ms") - latest = pd.Timestamp(datetime.datetime.utcnow() + timedelta(seconds=1)).round("ms") + now = pd.Timestamp(_utc_now()).round("ms") + hour_ago = pd.Timestamp(_utc_now() - timedelta(hours=1)).round("ms") + latest = pd.Timestamp(_utc_now() + timedelta(seconds=1)).round("ms") data = { "id": [123, 567, 890], @@ -221,7 +222,7 @@ def test_write_to_online_store_event_check(environment): # writes to online store via datasource (dataframe_source) materialization fs.materialize( start_date=datetime.datetime.now() - timedelta(hours=12), - end_date=datetime.datetime.utcnow(), + end_date=_utc_now(), ) df = fs.get_online_features( @@ -250,8 +251,8 @@ def test_write_to_online_store(environment, universal_data_sources): "conv_rate": [0.85], "acc_rate": [0.91], "avg_daily_trips": [14], - "event_timestamp": [pd.Timestamp(datetime.datetime.utcnow()).round("ms")], - "created": [pd.Timestamp(datetime.datetime.utcnow()).round("ms")], + "event_timestamp": [pd.Timestamp(_utc_now()).round("ms")], + "created": [pd.Timestamp(_utc_now()).round("ms")], } df_data = pd.DataFrame(data) diff --git a/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py b/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py index ce960b9c35..151f629289 100644 --- a/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py +++ b/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py @@ -1,5 +1,3 @@ -from datetime import datetime - import pandas as pd import pytest @@ -7,6 +5,7 @@ from feast.errors import SpecifiedFeaturesNotPresentError from feast.infra.offline_stores.file_source import FileSource from feast.types import Float64 +from feast.utils import _utc_now from tests.integration.feature_repos.universal.entities import customer, driver, item from tests.integration.feature_repos.universal.feature_views import ( conv_rate_plus_100_feature_view, @@ -50,8 +49,8 @@ def test_infer_odfv_list_features(environment, infer_features, tmp_path): "item_id": [0], "embedding_float": [fake_embedding], "embedding_double": [fake_embedding], - "event_timestamp": [pd.Timestamp(datetime.utcnow())], - "created": [pd.Timestamp(datetime.utcnow())], + "event_timestamp": [pd.Timestamp(_utc_now())], + "created": [pd.Timestamp(_utc_now())], } ) output_path = f"{tmp_path}/items.parquet" diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index c119ae800a..c06ccf2d4d 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -14,7 +14,7 @@ import logging import os import time -from datetime import datetime, timedelta +from datetime import timedelta from tempfile import mkstemp from unittest import mock @@ -46,6 +46,7 @@ from feast.repo_config import RegistryConfig from feast.stream_feature_view import Aggregation, StreamFeatureView from feast.types import Array, Bytes, Float32, Int32, Int64, String +from feast.utils import _utc_now from feast.value_type import ValueType from tests.integration.feature_repos.universal.entities import driver @@ -745,7 +746,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: ) # Simulate materialization - current_date = datetime.utcnow() + current_date = _utc_now() end_date = current_date.replace(tzinfo=utc) start_date = (current_date - timedelta(days=1)).replace(tzinfo=utc) test_registry.apply_materialization(feature_view, project, start_date, end_date) @@ -814,7 +815,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: ) # Simulate materialization a second time - current_date = datetime.utcnow() + current_date = _utc_now() end_date_1 = current_date.replace(tzinfo=utc) start_date_1 = (current_date - timedelta(days=1)).replace(tzinfo=utc) test_registry.apply_materialization( diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index ca15681c9b..928d05ad31 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -20,6 +20,7 @@ String, UnixTimestamp, ) +from feast.utils import _utc_now from tests.data.data_creator import create_basic_driver_dataset from tests.integration.feature_repos.universal.entities import driver from tests.integration.feature_repos.universal.feature_views import driver_feature_view @@ -93,7 +94,7 @@ def test_feature_get_historical_features_types_match( entity_df = pd.DataFrame() entity_df["driver_id"] = [1, 3] - ts = pd.Timestamp(datetime.utcnow()).round("ms") + ts = pd.Timestamp(_utc_now()).round("ms") entity_df["ts"] = [ ts - timedelta(hours=4), ts - timedelta(hours=2), diff --git a/sdk/python/tests/unit/cli/test_cli_chdir.py b/sdk/python/tests/unit/cli/test_cli_chdir.py index 12ca8f6b08..dd592db074 100644 --- a/sdk/python/tests/unit/cli/test_cli_chdir.py +++ b/sdk/python/tests/unit/cli/test_cli_chdir.py @@ -1,7 +1,8 @@ import tempfile -from datetime import datetime, timedelta +from datetime import timedelta from pathlib import Path +from feast.utils import _utc_now from tests.utils.cli_repo_creator import CliRunner @@ -29,7 +30,7 @@ def test_cli_chdir() -> None: ) assert result.returncode == 0 - end_date = datetime.utcnow() + end_date = _utc_now() start_date = end_date - timedelta(days=100) result = runner.run( [ diff --git a/sdk/python/tests/unit/local_feast_tests/test_init.py b/sdk/python/tests/unit/local_feast_tests/test_init.py index c5d3cbe57d..4543a23979 100644 --- a/sdk/python/tests/unit/local_feast_tests/test_init.py +++ b/sdk/python/tests/unit/local_feast_tests/test_init.py @@ -1,8 +1,9 @@ import tempfile -from datetime import datetime, timedelta +from datetime import timedelta from pathlib import Path from textwrap import dedent +from feast.utils import _utc_now from tests.utils.cli_repo_creator import CliRunner @@ -20,7 +21,7 @@ def test_repo_init() -> None: result = runner.run(["apply"], cwd=repo_path) assert result.returncode == 0 - end_date = datetime.utcnow() + end_date = _utc_now() start_date = end_date - timedelta(days=100) result = runner.run( ["materialize", start_date.isoformat(), end_date.isoformat()], cwd=repo_path diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 1e8cf45dcc..0b552c0453 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -3,7 +3,6 @@ import sqlite3 import sys import time -from datetime import datetime import numpy as np import pandas as pd @@ -17,6 +16,7 @@ from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RegistryConfig +from feast.utils import _utc_now from tests.integration.feature_repos.universal.feature_views import TAGS from tests.utils.cli_repo_creator import CliRunner, get_example_repo @@ -51,8 +51,8 @@ def test_get_online_features() -> None: "lat": ValueProto(double_val=0.1), "lon": ValueProto(string_val="1.0"), }, - datetime.utcnow(), - datetime.utcnow(), + _utc_now(), + _utc_now(), ) ], progress=None, @@ -72,8 +72,8 @@ def test_get_online_features() -> None: "name": ValueProto(string_val="John"), "age": ValueProto(int64_val=3), }, - datetime.utcnow(), - datetime.utcnow(), + _utc_now(), + _utc_now(), ) ], progress=None, @@ -90,8 +90,8 @@ def test_get_online_features() -> None: ( customer_key, {"trips": ValueProto(int64_val=7)}, - datetime.utcnow(), - datetime.utcnow(), + _utc_now(), + _utc_now(), ) ], progress=None, @@ -318,8 +318,8 @@ def test_online_to_df(): "lat": ValueProto(double_val=d * lat_multiply), "lon": ValueProto(string_val=str(d * lon_multiply)), }, - datetime.utcnow(), - datetime.utcnow(), + _utc_now(), + _utc_now(), ) ], progress=None, @@ -348,8 +348,8 @@ def test_online_to_df(): "name": ValueProto(string_val=name + str(c)), "age": ValueProto(int64_val=c * age_multiply), }, - datetime.utcnow(), - datetime.utcnow(), + _utc_now(), + _utc_now(), ) ], progress=None, @@ -372,8 +372,8 @@ def test_online_to_df(): ( combo_keys, {"trips": ValueProto(int64_val=c * d)}, - datetime.utcnow(), - datetime.utcnow(), + _utc_now(), + _utc_now(), ) ], progress=None, @@ -468,8 +468,8 @@ def test_sqlite_get_online_documents() -> None: ) ) }, - datetime.utcnow(), - datetime.utcnow(), + _utc_now(), + _utc_now(), ) ) @@ -488,7 +488,7 @@ def test_sqlite_get_online_documents() -> None: ) for i in range(n) ], - "event_timestamp": [datetime.utcnow() for _ in range(n)], + "event_timestamp": [_utc_now() for _ in range(n)], } ) diff --git a/sdk/python/tests/unit/test_datetime.py b/sdk/python/tests/unit/test_datetime.py new file mode 100644 index 0000000000..aaab507ed0 --- /dev/null +++ b/sdk/python/tests/unit/test_datetime.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- + + +""" +Test the retirement of datetime.utcnow() function. +""" diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index b387f55d8b..981968df0d 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -1,9 +1,8 @@ -from datetime import datetime, timedelta +from datetime import timedelta import pytest from typeguard import TypeCheckError -from feast import utils from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat from feast.data_source import KafkaSource @@ -13,6 +12,7 @@ from feast.infra.offline_stores.file_source import FileSource from feast.protos.feast.types.Value_pb2 import ValueType from feast.types import Float32 +from feast.utils import _utc_now, make_tzaware def test_create_feature_view_with_conflicting_entities(): @@ -143,9 +143,9 @@ def test_update_materialization_intervals(): ) assert len(updated_feature_view.materialization_intervals) == 0 - current_time = datetime.utcnow() - start_date = utils.make_tzaware(current_time - timedelta(days=1)) - end_date = utils.make_tzaware(current_time) + current_time = _utc_now() + start_date = make_tzaware(current_time - timedelta(days=1)) + end_date = make_tzaware(current_time) updated_feature_view.materialization_intervals.append((start_date, end_date)) # Update the Feature View, i.e. simply update the name diff --git a/sdk/python/tests/unit/test_stream_feature_view.py b/sdk/python/tests/unit/test_stream_feature_view.py index 77431666c3..4f93691028 100644 --- a/sdk/python/tests/unit/test_stream_feature_view.py +++ b/sdk/python/tests/unit/test_stream_feature_view.py @@ -1,9 +1,8 @@ import copy -from datetime import datetime, timedelta +from datetime import timedelta import pytest -from feast import utils from feast.aggregation import Aggregation from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat @@ -16,6 +15,7 @@ ) from feast.stream_feature_view import StreamFeatureView, stream_feature_view from feast.types import Float32 +from feast.utils import _utc_now, make_tzaware def test_create_batch_feature_view(): @@ -286,9 +286,9 @@ def test_update_materialization_intervals(): udf=simple_udf, tags={}, ) - current_time = datetime.utcnow() - start_date = utils.make_tzaware(current_time - timedelta(days=1)) - end_date = utils.make_tzaware(current_time) + current_time = _utc_now() + start_date = make_tzaware(current_time - timedelta(days=1)) + end_date = make_tzaware(current_time) stored_stream_feature_view.materialization_intervals.append((start_date, end_date)) # Update the stream feature view i.e. here it's simply the name diff --git a/sdk/python/tests/utils/basic_read_write_test.py b/sdk/python/tests/utils/basic_read_write_test.py index 5a93a05a1f..c09a94083f 100644 --- a/sdk/python/tests/utils/basic_read_write_test.py +++ b/sdk/python/tests/utils/basic_read_write_test.py @@ -1,9 +1,10 @@ -from datetime import datetime, timedelta +from datetime import timedelta from typing import Optional from feast.feature_store import FeatureStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.utils import _utc_now def basic_rw_test( @@ -65,13 +66,13 @@ def _driver_rw_test(event_ts, created_ts, write, expect_read): """ 1. Basic test: write value, read it back """ - time_1 = datetime.utcnow() + time_1 = _utc_now() _driver_rw_test( event_ts=time_1, created_ts=time_1, write=(1.1, "3.1"), expect_read=(1.1, "3.1") ) """ Values with an new event_ts should overwrite older ones """ - time_3 = datetime.utcnow() + time_3 = _utc_now() _driver_rw_test( event_ts=time_1 + timedelta(hours=1), created_ts=time_3, diff --git a/sdk/python/tests/utils/dynamo_table_creator.py b/sdk/python/tests/utils/dynamo_table_creator.py index 20bac122b3..0ebc939dc1 100644 --- a/sdk/python/tests/utils/dynamo_table_creator.py +++ b/sdk/python/tests/utils/dynamo_table_creator.py @@ -1,11 +1,10 @@ -from datetime import datetime - import boto3 from feast import utils from feast.infra.online_stores.helpers import compute_entity_id from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.utils import _utc_now def create_n_customer_test_samples(n=10): @@ -19,7 +18,7 @@ def create_n_customer_test_samples(n=10): "name": ValueProto(string_val="John"), "age": ValueProto(int64_val=3), }, - datetime.utcnow(), + _utc_now(), None, ) for i in range(n) diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index d9104bae42..1a8bedc796 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -10,6 +10,7 @@ from pytz import utc from feast import FeatureStore, FeatureView, RepoConfig +from feast.utils import _utc_now from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, ) @@ -31,7 +32,7 @@ def validate_offline_online_store_consistency( fs: FeatureStore, fv: FeatureView, split_dt: datetime ) -> None: - now = datetime.utcnow() + now = _utc_now() full_feature_names = True check_offline_store: bool = True diff --git a/sdk/python/tests/utils/online_write_benchmark.py b/sdk/python/tests/utils/online_write_benchmark.py index 8a138f41db..9b1a4eb0b2 100644 --- a/sdk/python/tests/utils/online_write_benchmark.py +++ b/sdk/python/tests/utils/online_write_benchmark.py @@ -2,7 +2,7 @@ import random import string import tempfile -from datetime import datetime, timedelta +from datetime import timedelta import click import pyarrow as pa @@ -16,7 +16,7 @@ from feast.field import Field from feast.repo_config import RepoConfig from feast.types import Float32, Int32 -from feast.utils import _convert_arrow_to_proto +from feast.utils import _convert_arrow_to_proto, _utc_now def create_driver_hourly_stats_feature_view(source): @@ -69,7 +69,7 @@ def benchmark_writes(): provider = store._get_provider() - end_date = datetime.utcnow() + end_date = _utc_now() start_date = end_date - timedelta(days=14) customers = list(range(100)) data = create_driver_hourly_stats_df(customers, start_date, end_date) diff --git a/sdk/python/tests/utils/test_log_creator.py b/sdk/python/tests/utils/test_log_creator.py index ec0d92814c..f072f4c886 100644 --- a/sdk/python/tests/utils/test_log_creator.py +++ b/sdk/python/tests/utils/test_log_creator.py @@ -8,12 +8,12 @@ import numpy as np import pandas as pd import pyarrow -import pytz from feast import FeatureService, FeatureStore, FeatureView from feast.errors import FeatureViewNotFoundException from feast.feature_logging import LOG_DATE_FIELD, LOG_TIMESTAMP_FIELD, REQUEST_ID_FIELD from feast.protos.feast.serving.ServingService_pb2 import FieldStatus +from feast.utils import _utc_now def get_latest_rows( @@ -64,9 +64,7 @@ def generate_expected_logs( logs[f"{col}__status"] = FieldStatus.PRESENT if feature_view.ttl: logs[f"{col}__status"] = logs[f"{col}__status"].mask( - df[timestamp_column] - < datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - - feature_view.ttl, + df[timestamp_column] < _utc_now() - feature_view.ttl, FieldStatus.OUTSIDE_MAX_AGE, ) @@ -119,7 +117,7 @@ def prepare_logs( f"{destination_field}__status" ].mask( logs_df[f"{destination_field}__timestamp"] - < (datetime.datetime.utcnow() - view.ttl), + < (_utc_now() - view.ttl), FieldStatus.OUTSIDE_MAX_AGE, ) From b0dc6832ff446429390a916aa9e0e61066cbde1d Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Thu, 4 Jul 2024 12:39:46 -0400 Subject: [PATCH 09/33] chore: Upgrading sqlite_vec to latest package (#4332) Signed-off-by: Francisco Javier Arceo Co-authored-by: Francisco Javier Arceo --- infra/scripts/pixi/pixi.lock | 306 ++++++++++++++++++ infra/scripts/pixi/pixi.toml | 2 +- .../requirements/py3.10-ci-requirements.txt | 2 + .../requirements/py3.10-requirements.txt | 2 + .../requirements/py3.11-ci-requirements.txt | 2 + .../requirements/py3.11-requirements.txt | 2 + .../requirements/py3.9-ci-requirements.txt | 2 + .../requirements/py3.9-requirements.txt | 2 + 8 files changed, 319 insertions(+), 1 deletion(-) diff --git a/infra/scripts/pixi/pixi.lock b/infra/scripts/pixi/pixi.lock index f1ce2d2658..1ca8742026 100644 --- a/infra/scripts/pixi/pixi.lock +++ b/infra/scripts/pixi/pixi.lock @@ -11,6 +11,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + osx-64: + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda @@ -41,6 +44,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-64: + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h5846eda_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.3.1-h87427d6_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.10.14-h00d2728_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda @@ -84,6 +102,22 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-64: + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.6.2-h73e2aa4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h5846eda_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.3.1-h87427d6_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.9-h657bba9_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda @@ -127,6 +161,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-64: + - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h5846eda_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.3.1-h87427d6_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.9.19-h7a9c478_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda @@ -172,6 +221,19 @@ packages: license_family: BSD size: 23621 timestamp: 1650670423406 +- kind: conda + name: bzip2 + version: 1.0.8 + build: h10d778d_5 + build_number: 5 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda + sha256: 61fb2b488928a54d9472113e1280b468a309561caa54f33825a3593da390b242 + md5: 6097a6ca9ada32699b5fc4312dd6ef18 + license: bzip2-1.0.6 + license_family: BSD + size: 127885 + timestamp: 1699280178474 - kind: conda name: bzip2 version: 1.0.8 @@ -222,6 +284,17 @@ packages: license: ISC size: 155725 timestamp: 1706844034242 +- kind: conda + name: ca-certificates + version: 2024.7.4 + build: h8857fd0_0 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda + sha256: d16f46c489cb3192305c7d25b795333c5fc17bb0986de20598ed519f8c9cc9e4 + md5: 7df874a4b05b2d2b82826190170eaa0f + license: ISC + size: 154473 + timestamp: 1720077510541 - kind: conda name: ld_impl_linux-64 version: '2.40' @@ -264,6 +337,21 @@ packages: license_family: Apache size: 1248885 timestamp: 1715020154867 +- kind: conda + name: libcxx + version: 17.0.6 + build: heb59cac_3 + build_number: 3 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + sha256: 9df841c64b19a3843869467ff8ff2eb3f6c5491ebaac8fd94fb8029a5b00dcbf + md5: ef15f182e353155497e13726b915bfc4 + depends: + - __osx >=10.13 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + size: 1250659 + timestamp: 1720040263499 - kind: conda name: libexpat version: 2.6.2 @@ -280,6 +368,20 @@ packages: license_family: MIT size: 73730 timestamp: 1710362120304 +- kind: conda + name: libexpat + version: 2.6.2 + build: h73e2aa4_0 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.6.2-h73e2aa4_0.conda + sha256: a188a77b275d61159a32ab547f7d17892226e7dac4518d2c6ac3ac8fc8dfde92 + md5: 3d1d51c8f716d97c864d12f7af329526 + constrains: + - expat 2.6.2.* + license: MIT + license_family: MIT + size: 69246 + timestamp: 1710362566073 - kind: conda name: libexpat version: 2.6.2 @@ -294,6 +396,19 @@ packages: license_family: MIT size: 63655 timestamp: 1710362424980 +- kind: conda + name: libffi + version: 3.4.2 + build: h0d85af4_5 + build_number: 5 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 + sha256: 7a2d27a936ceee6942ea4d397f9c7d136f12549d86f7617e8b6bad51e01a941f + md5: ccb34fb14960ad8b125962d3d79b31a9 + license: MIT + license_family: MIT + size: 51348 + timestamp: 1636488394370 - kind: conda name: libffi version: 3.4.2 @@ -429,6 +544,20 @@ packages: license: Unlicense size: 859858 timestamp: 1713367435849 +- kind: conda + name: libsqlite + version: 3.46.0 + build: h1b8f9f3_0 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda + sha256: 63af1a9e3284c7e4952364bafe7267e41e2d9d8bcc0e85a4ea4b0ec02d3693f6 + md5: 5dadfbc1a567fe6e475df4ce3148be09 + depends: + - __osx >=10.13 + - libzlib >=1.2.13,<2.0a0 + license: Unlicense + size: 908643 + timestamp: 1718050720117 - kind: conda name: libstdcxx-ng version: 13.2.0 @@ -504,6 +633,23 @@ packages: license_family: Other size: 46768 timestamp: 1716874151980 +- kind: conda + name: libzlib + version: 1.3.1 + build: h87427d6_1 + build_number: 1 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda + sha256: 80a62db652b1da0ccc100812a1d86e94f75028968991bfb17f9536f3aa72d91d + md5: b7575b5aa92108dcc9aaab0f05f2dbce + depends: + - __osx >=10.13 + constrains: + - zlib 1.3.1 *_1 + license: Zlib + license_family: Other + size: 57372 + timestamp: 1716874211519 - kind: conda name: ncurses version: 6.4.20240210 @@ -517,6 +663,17 @@ packages: license: X11 AND BSD-3-Clause size: 895669 timestamp: 1710866638986 +- kind: conda + name: ncurses + version: '6.5' + build: h5846eda_0 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h5846eda_0.conda + sha256: 6ecc73db0e49143092c0934355ac41583a5d5a48c6914c5f6ca48e562d3a4b79 + md5: 02a888433d165c99bf09784a7b14d900 + license: X11 AND BSD-3-Clause + size: 823601 + timestamp: 1715195267791 - kind: conda name: ncurses version: '6.5' @@ -581,6 +738,24 @@ packages: license_family: Apache size: 2893954 timestamp: 1716468329572 +- kind: conda + name: openssl + version: 3.3.1 + build: h87427d6_1 + build_number: 1 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.3.1-h87427d6_1.conda + sha256: 60eed5d771207bcef05e0547c8f93a61d0ad1dcf75e19f8f8d9ded8094d78477 + md5: d838ffe9ec3c6d971f110e04487466ff + depends: + - __osx >=10.13 + - ca-certificates + constrains: + - pyopenssl >=22.1 + license: Apache-2.0 + license_family: Apache + size: 2551950 + timestamp: 1719364820943 - kind: conda name: python version: 3.9.19 @@ -610,6 +785,30 @@ packages: license: Python-2.0 size: 23800555 timestamp: 1710940120866 +- kind: conda + name: python + version: 3.9.19 + build: h7a9c478_0_cpython + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/python-3.9.19-h7a9c478_0_cpython.conda + sha256: 58b76be84683bc03112b3ed7e377e99af24844ebf7d7568f6466a2dae7a887fe + md5: 7d53d366acd9dbfb498c69326ccb520a + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.9.* *_cp39 + license: Python-2.0 + size: 12372436 + timestamp: 1710940037648 - kind: conda name: python version: 3.9.19 @@ -634,6 +833,30 @@ packages: license: Python-2.0 size: 11847835 timestamp: 1710939779164 +- kind: conda + name: python + version: 3.10.14 + build: h00d2728_0_cpython + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/python-3.10.14-h00d2728_0_cpython.conda + sha256: 00c1de2d46ede26609ef4e84a44b83be7876ba6a0215b7c83bff41a0656bf694 + md5: 0a1cddc4382c5c171e791c70740546dd + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.10.* *_cp310 + license: Python-2.0 + size: 11890228 + timestamp: 1710940046031 - kind: conda name: python version: 3.10.14 @@ -687,6 +910,32 @@ packages: license: Python-2.0 size: 25517742 timestamp: 1710939725109 +- kind: conda + name: python + version: 3.11.9 + build: h657bba9_0_cpython + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.9-h657bba9_0_cpython.conda + sha256: 3b50a5abb3b812875beaa9ab792dbd1bf44f335c64e9f9fedcf92d953995651c + md5: 612763bc5ede9552e4233ec518b9c9fb + depends: + - __osx >=10.9 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.6.2,<3.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.3,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 15503226 + timestamp: 1713553747073 - kind: conda name: python version: 3.11.9 @@ -774,6 +1023,36 @@ packages: license_family: GPL size: 250351 timestamp: 1679532511311 +- kind: conda + name: readline + version: '8.2' + build: h9e318b2_1 + build_number: 1 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda + sha256: 41e7d30a097d9b060037f0c6a2b1d4c4ae7e942c06c943d23f9d481548478568 + md5: f17f77f2acf4d344734bda76829ce14e + depends: + - ncurses >=6.3,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 255870 + timestamp: 1679532707590 +- kind: conda + name: tk + version: 8.6.13 + build: h1abcd95_1 + build_number: 1 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda + sha256: 30412b2e9de4ff82d8c2a7e5d06a15f4f4fef1809a72138b6ccb53a33b26faf5 + md5: bf830ba5afc507c6232d4ef0fb1a882d + depends: + - libzlib >=1.2.13,<2.0.0a0 + license: TCL + license_family: BSD + size: 3270220 + timestamp: 1699202389792 - kind: conda name: tk version: 8.6.13 @@ -831,6 +1110,22 @@ packages: license: Apache-2.0 OR MIT size: 11891252 timestamp: 1714233659570 +- kind: conda + name: uv + version: 0.1.45 + build: h4e38c46_0 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + sha256: 8c11774ca1940dcd90187ce240afea26b76e2942f9b18d65f6d4b483534193fd + md5: 754ce8a22c94a30c7bbd42274c7fae31 + depends: + - __osx >=10.13 + - libcxx >=16 + constrains: + - __osx >=10.12 + license: Apache-2.0 OR MIT + size: 8937335 + timestamp: 1716265195083 - kind: conda name: uv version: 0.1.45 @@ -871,3 +1166,14 @@ packages: license: LGPL-2.1 and GPL-2.0 size: 235693 timestamp: 1660346961024 +- kind: conda + name: xz + version: 5.2.6 + build: h775f41a_0 + subdir: osx-64 + url: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 + sha256: eb09823f34cc2dd663c0ec4ab13f246f45dcd52e5b8c47b9864361de5204a1c8 + md5: a72f9d4ea13d55d745ff1ed594747f10 + license: LGPL-2.1 and GPL-2.0 + size: 238119 + timestamp: 1660346964847 diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml index 10179339f7..487c6f7def 100644 --- a/infra/scripts/pixi/pixi.toml +++ b/infra/scripts/pixi/pixi.toml @@ -1,7 +1,7 @@ [project] name = "pixi-feast" channels = ["conda-forge"] -platforms = ["linux-64", "osx-arm64"] +platforms = ["linux-64", "osx-arm64", "osx-64"] [tasks] diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 3aa7130ccf..0eaababc0d 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -278,6 +278,8 @@ googleapis-common-protos[grpc]==1.63.2 # grpcio-status great-expectations==0.18.16 # via feast (setup.py) +greenlet==3.0.3 + # via sqlalchemy grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 72124636b6..308123600c 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -50,6 +50,8 @@ fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask +greenlet==3.0.3 + # via sqlalchemy gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 673047b5c7..d0663a2bea 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -269,6 +269,8 @@ googleapis-common-protos[grpc]==1.63.2 # grpcio-status great-expectations==0.18.16 # via feast (setup.py) +greenlet==3.0.3 + # via sqlalchemy grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 408f392515..f21afdb5b1 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -48,6 +48,8 @@ fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask +greenlet==3.0.3 + # via sqlalchemy gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 83009f8730..f09c666f42 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -278,6 +278,8 @@ googleapis-common-protos[grpc]==1.63.2 # grpcio-status great-expectations==0.18.16 # via feast (setup.py) +greenlet==3.0.3 + # via sqlalchemy grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 3c833438de..52ff8a0f4f 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -50,6 +50,8 @@ fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask +greenlet==3.0.3 + # via sqlalchemy gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 From 0d89d1519fc6b8ddd05a2588138e2e85f5a921b1 Mon Sep 17 00:00:00 2001 From: Tom Steenbergen <41334387+TomSteenbergen@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:27:39 +0200 Subject: [PATCH 10/33] fix: Remove redundant batching in PostgreSQLOnlineStore.online_write_batch and fix progress bar (#4331) * Remove batching and fix tqdm progress bar Signed-off-by: TomSteenbergen * Comment Signed-off-by: TomSteenbergen * Remove test changes Signed-off-by: TomSteenbergen * Update comment Signed-off-by: TomSteenbergen --------- Signed-off-by: TomSteenbergen --- .../feast/infra/online_stores/contrib/postgres.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 8715f0f65b..330b50bc78 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -75,7 +75,6 @@ def online_write_batch( Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] ], progress: Optional[Callable[[int], Any]], - batch_size: int = 5000, ) -> None: # Format insert values insert_values = [] @@ -118,15 +117,13 @@ def online_write_batch( """ ).format(sql.Identifier(_table_id(config.project, table))) - # Push data in batches to online store + # Push data into the online store with self._get_conn(config) as conn, conn.cursor() as cur: - for i in range(0, len(insert_values), batch_size): - cur_batch = insert_values[i : i + batch_size] - cur.executemany(sql_query, cur_batch) - conn.commit() + cur.executemany(sql_query, insert_values) + conn.commit() - if progress: - progress(len(cur_batch)) + if progress: + progress(len(data)) def online_read( self, From cea52e9fb02cb9e0b8f48206278474f5a5fa167e Mon Sep 17 00:00:00 2001 From: Tom Steenbergen <41334387+TomSteenbergen@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:29:43 +0200 Subject: [PATCH 11/33] feat: Add async feature retrieval for Postgres Online Store (#4327) * Add async retrieval for postgres Signed-off-by: TomSteenbergen * Format Signed-off-by: TomSteenbergen * Update _prepare_keys method Signed-off-by: TomSteenbergen * Fix typo Signed-off-by: TomSteenbergen --------- Signed-off-by: TomSteenbergen --- .../infra/online_stores/contrib/postgres.py | 186 ++++++++++++------ .../infra/utils/postgres/connection_utils.py | 25 ++- .../online_store/test_universal_online.py | 2 +- 3 files changed, 150 insertions(+), 63 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 330b50bc78..ff73a4a347 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -4,6 +4,7 @@ from datetime import datetime from typing import ( Any, + AsyncGenerator, Callable, Dict, Generator, @@ -12,18 +13,24 @@ Optional, Sequence, Tuple, + Union, ) import pytz -from psycopg import sql +from psycopg import AsyncConnection, sql from psycopg.connection import Connection -from psycopg_pool import ConnectionPool +from psycopg_pool import AsyncConnectionPool, ConnectionPool from feast import Entity from feast.feature_view import FeatureView from feast.infra.key_encoding_utils import get_list_val_str, serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore -from feast.infra.utils.postgres.connection_utils import _get_conn, _get_connection_pool +from feast.infra.utils.postgres.connection_utils import ( + _get_conn, + _get_conn_async, + _get_connection_pool, + _get_connection_pool_async, +) from feast.infra.utils.postgres.postgres_config import ConnectionType, PostgreSQLConfig from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -51,6 +58,9 @@ class PostgreSQLOnlineStore(OnlineStore): _conn: Optional[Connection] = None _conn_pool: Optional[ConnectionPool] = None + _conn_async: Optional[AsyncConnection] = None + _conn_pool_async: Optional[AsyncConnectionPool] = None + @contextlib.contextmanager def _get_conn(self, config: RepoConfig) -> Generator[Connection, Any, Any]: assert config.online_store.type == "postgres" @@ -67,6 +77,24 @@ def _get_conn(self, config: RepoConfig) -> Generator[Connection, Any, Any]: self._conn = _get_conn(config.online_store) yield self._conn + @contextlib.asynccontextmanager + async def _get_conn_async( + self, config: RepoConfig + ) -> AsyncGenerator[AsyncConnection, Any]: + if config.online_store.conn_type == ConnectionType.pool: + if not self._conn_pool_async: + self._conn_pool_async = await _get_connection_pool_async( + config.online_store + ) + await self._conn_pool_async.open() + connection = await self._conn_pool_async.getconn() + yield connection + await self._conn_pool_async.putconn(connection) + else: + if not self._conn_async: + self._conn_async = await _get_conn_async(config.online_store) + yield self._conn_async + def online_write_batch( self, config: RepoConfig, @@ -132,69 +160,107 @@ def online_read( entity_keys: List[EntityKeyProto], requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: - result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + keys = self._prepare_keys(entity_keys, config.entity_key_serialization_version) + query, params = self._construct_query_and_params( + config, table, keys, requested_features + ) - project = config.project with self._get_conn(config) as conn, conn.cursor() as cur: - # Collecting all the keys to a list allows us to make fewer round trips - # to PostgreSQL - keys = [] - for entity_key in entity_keys: - keys.append( - serialize_entity_key( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) - ) + cur.execute(query, params) + rows = cur.fetchall() - if not requested_features: - cur.execute( - sql.SQL( - """ - SELECT entity_key, feature_name, value, event_ts - FROM {} WHERE entity_key = ANY(%s); - """ - ).format( - sql.Identifier(_table_id(project, table)), - ), - (keys,), - ) - else: - cur.execute( - sql.SQL( - """ - SELECT entity_key, feature_name, value, event_ts - FROM {} WHERE entity_key = ANY(%s) and feature_name = ANY(%s); - """ - ).format( - sql.Identifier(_table_id(project, table)), - ), - (keys, requested_features), - ) + return self._process_rows(keys, rows) - rows = cur.fetchall() + async def online_read_async( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + keys = self._prepare_keys(entity_keys, config.entity_key_serialization_version) + query, params = self._construct_query_and_params( + config, table, keys, requested_features + ) - # Since we don't know the order returned from PostgreSQL we'll need - # to construct a dict to be able to quickly look up the correct row - # when we iterate through the keys since they are in the correct order - values_dict = defaultdict(list) - for row in rows if rows is not None else []: - values_dict[ - row[0] if isinstance(row[0], bytes) else row[0].tobytes() - ].append(row[1:]) - - for key in keys: - if key in values_dict: - value = values_dict[key] - res = {} - for feature_name, value_bin, event_ts in value: - val = ValueProto() - val.ParseFromString(bytes(value_bin)) - res[feature_name] = val - result.append((event_ts, res)) - else: - result.append((None, None)) + async with self._get_conn_async(config) as conn: + async with conn.cursor() as cur: + await cur.execute(query, params) + rows = await cur.fetchall() + + return self._process_rows(keys, rows) + + @staticmethod + def _construct_query_and_params( + config: RepoConfig, + table: FeatureView, + keys: List[bytes], + requested_features: Optional[List[str]] = None, + ) -> Tuple[sql.Composed, Union[Tuple[List[bytes], List[str]], Tuple[List[bytes]]]]: + """Construct the SQL query based on the given parameters.""" + if requested_features: + query = sql.SQL( + """ + SELECT entity_key, feature_name, value, event_ts + FROM {} WHERE entity_key = ANY(%s) AND feature_name = ANY(%s); + """ + ).format( + sql.Identifier(_table_id(config.project, table)), + ) + params = (keys, requested_features) + else: + query = sql.SQL( + """ + SELECT entity_key, feature_name, value, event_ts + FROM {} WHERE entity_key = ANY(%s); + """ + ).format( + sql.Identifier(_table_id(config.project, table)), + ) + params = (keys, []) + return query, params + + @staticmethod + def _prepare_keys( + entity_keys: List[EntityKeyProto], entity_key_serialization_version: int + ) -> List[bytes]: + """Prepare all keys in a list to make fewer round trips to the database.""" + return [ + serialize_entity_key( + entity_key, + entity_key_serialization_version=entity_key_serialization_version, + ) + for entity_key in entity_keys + ] + + @staticmethod + def _process_rows( + keys: List[bytes], rows: List[Tuple] + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + """Transform the retrieved rows in the desired output. + PostgreSQL may return rows in an unpredictable order. Therefore, `values_dict` + is created to quickly look up the correct row using the keys, since these are + actually in the correct order. + """ + values_dict = defaultdict(list) + for row in rows if rows is not None else []: + values_dict[ + row[0] if isinstance(row[0], bytes) else row[0].tobytes() + ].append(row[1:]) + + result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + for key in keys: + if key in values_dict: + value = values_dict[key] + res = {} + for feature_name, value_bin, event_ts in value: + val = ValueProto() + val.ParseFromString(bytes(value_bin)) + res[feature_name] = val + result.append((event_ts, res)) + else: + result.append((None, None)) return result def update( diff --git a/sdk/python/feast/infra/utils/postgres/connection_utils.py b/sdk/python/feast/infra/utils/postgres/connection_utils.py index e0599019b9..7b37ea981f 100644 --- a/sdk/python/feast/infra/utils/postgres/connection_utils.py +++ b/sdk/python/feast/infra/utils/postgres/connection_utils.py @@ -4,8 +4,8 @@ import pandas as pd import psycopg import pyarrow as pa -from psycopg.connection import Connection -from psycopg_pool import ConnectionPool +from psycopg import AsyncConnection, Connection +from psycopg_pool import AsyncConnectionPool, ConnectionPool from feast.infra.utils.postgres.postgres_config import PostgreSQLConfig from feast.type_map import arrow_to_pg_type @@ -21,6 +21,16 @@ def _get_conn(config: PostgreSQLConfig) -> Connection: return conn +async def _get_conn_async(config: PostgreSQLConfig) -> AsyncConnection: + """Get a psycopg `AsyncConnection`.""" + conn = await psycopg.AsyncConnection.connect( + conninfo=_get_conninfo(config), + keepalives_idle=config.keepalives_idle, + **_get_conn_kwargs(config), + ) + return conn + + def _get_connection_pool(config: PostgreSQLConfig) -> ConnectionPool: """Get a psycopg `ConnectionPool`.""" return ConnectionPool( @@ -32,6 +42,17 @@ def _get_connection_pool(config: PostgreSQLConfig) -> ConnectionPool: ) +async def _get_connection_pool_async(config: PostgreSQLConfig) -> AsyncConnectionPool: + """Get a psycopg `AsyncConnectionPool`.""" + return AsyncConnectionPool( + conninfo=_get_conninfo(config), + min_size=config.min_conn, + max_size=config.max_conn, + open=False, + kwargs=_get_conn_kwargs(config), + ) + + def _get_conninfo(config: PostgreSQLConfig) -> str: """Get the `conninfo` argument required for connection objects.""" return ( diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 38656b90a9..2ffe869ef5 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -488,7 +488,7 @@ def test_online_retrieval_with_event_timestamps(environment, universal_data_sour @pytest.mark.integration -@pytest.mark.universal_online_stores(only=["redis", "dynamodb"]) +@pytest.mark.universal_online_stores(only=["redis", "dynamodb", "postgres"]) def test_async_online_retrieval_with_event_timestamps( environment, universal_data_sources ): From f5697863669a6bb9dbd491f79192e8ddd0073388 Mon Sep 17 00:00:00 2001 From: stanconia <36575269+stanconia@users.noreply.github.com> Date: Tue, 9 Jul 2024 05:25:29 -0400 Subject: [PATCH 12/33] feat: Add Async refresh to Sql Registry (#4251) * Add sql registry async refresh Signed-off-by: Stanley Opara * make refresh code a daemon thread Signed-off-by: Stanley Opara * Change RegistryConfig to cacheMode Signed-off-by: Stanley Opara * Only run async when ttl > 0 Signed-off-by: Stanley Opara * make refresh async run in a loop Signed-off-by: Stanley Opara * make refresh async run in a loop Signed-off-by: Stanley Opara * Reorder async refresh call Signed-off-by: Stanley Opara * Add documentation Signed-off-by: Stanley Opara * Update test_universal_registry.py Signed-off-by: Stanley Opara * Force rerun of tests Signed-off-by: Stanley Opara * Force rerun of tests Signed-off-by: Stanley Opara * Format repo config file Signed-off-by: Stanley Opara --------- Signed-off-by: Stanley Opara Co-authored-by: Stanley Opara --- .../feast/infra/registry/caching_registry.py | 57 +++++---- sdk/python/feast/infra/registry/sql.py | 4 +- sdk/python/feast/repo_config.py | 3 + .../registration/test_universal_registry.py | 113 ++++++++++++++++-- 4 files changed, 145 insertions(+), 32 deletions(-) diff --git a/sdk/python/feast/infra/registry/caching_registry.py b/sdk/python/feast/infra/registry/caching_registry.py index f7eab7d70a..298639028d 100644 --- a/sdk/python/feast/infra/registry/caching_registry.py +++ b/sdk/python/feast/infra/registry/caching_registry.py @@ -1,4 +1,6 @@ +import atexit import logging +import threading from abc import abstractmethod from datetime import timedelta from threading import Lock @@ -21,11 +23,7 @@ class CachingRegistry(BaseRegistry): - def __init__( - self, - project: str, - cache_ttl_seconds: int, - ): + def __init__(self, project: str, cache_ttl_seconds: int, cache_mode: str): self.cached_registry_proto = self.proto() proto_registry_utils.init_project_metadata(self.cached_registry_proto, project) self.cached_registry_proto_created = _utc_now() @@ -33,6 +31,10 @@ def __init__( self.cached_registry_proto_ttl = timedelta( seconds=cache_ttl_seconds if cache_ttl_seconds is not None else 0 ) + self.cache_mode = cache_mode + if cache_mode == "thread": + self._start_thread_async_refresh(cache_ttl_seconds) + atexit.register(self._exit_handler) @abstractmethod def _get_data_source(self, name: str, project: str) -> DataSource: @@ -322,22 +324,35 @@ def refresh(self, project: Optional[str] = None): self.cached_registry_proto_created = _utc_now() def _refresh_cached_registry_if_necessary(self): - with self._refresh_lock: - expired = ( - self.cached_registry_proto is None - or self.cached_registry_proto_created is None - ) or ( - self.cached_registry_proto_ttl.total_seconds() - > 0 # 0 ttl means infinity - and ( - _utc_now() - > ( - self.cached_registry_proto_created - + self.cached_registry_proto_ttl + if self.cache_mode == "sync": + with self._refresh_lock: + expired = ( + self.cached_registry_proto is None + or self.cached_registry_proto_created is None + ) or ( + self.cached_registry_proto_ttl.total_seconds() + > 0 # 0 ttl means infinity + and ( + _utc_now() + > ( + self.cached_registry_proto_created + + self.cached_registry_proto_ttl + ) ) ) - ) + if expired: + logger.info("Registry cache expired, so refreshing") + self.refresh() + + def _start_thread_async_refresh(self, cache_ttl_seconds): + self.refresh() + if cache_ttl_seconds <= 0: + return + self.registry_refresh_thread = threading.Timer( + cache_ttl_seconds, self._start_thread_async_refresh, [cache_ttl_seconds] + ) + self.registry_refresh_thread.setDaemon(True) + self.registry_refresh_thread.start() - if expired: - logger.info("Registry cache expired, so refreshing") - self.refresh() + def _exit_handler(self): + self.registry_refresh_thread.cancel() diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 6ef08989b7..a2b16a3a09 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -193,7 +193,9 @@ def __init__( ) metadata.create_all(self.engine) super().__init__( - project=project, cache_ttl_seconds=registry_config.cache_ttl_seconds + project=project, + cache_ttl_seconds=registry_config.cache_ttl_seconds, + cache_mode=registry_config.cache_mode, ) def teardown(self): diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 8d6bff2818..137023ef22 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -124,6 +124,9 @@ class RegistryConfig(FeastBaseModel): sqlalchemy_config_kwargs: Dict[str, Any] = {} """ Dict[str, Any]: Extra arguments to pass to SQLAlchemy.create_engine. """ + cache_mode: StrictStr = "sync" + """ str: Cache mode type, Possible options are sync and thread(asynchronous caching using threading library)""" + @field_validator("path") def validate_path(cls, path: str, values: ValidationInfo) -> str: if values.data.get("registry_type") == "sql": diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index c06ccf2d4d..b0738c8419 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -125,7 +125,7 @@ def minio_registry() -> Registry: logger = logging.getLogger(__name__) -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def pg_registry(): container = ( DockerContainer("postgres:latest") @@ -137,6 +137,35 @@ def pg_registry(): container.start() + registry_config = _given_registry_config_for_pg_sql(container) + + yield SqlRegistry(registry_config, "project", None) + + container.stop() + + +@pytest.fixture(scope="function") +def pg_registry_async(): + container = ( + DockerContainer("postgres:latest") + .with_exposed_ports(5432) + .with_env("POSTGRES_USER", POSTGRES_USER) + .with_env("POSTGRES_PASSWORD", POSTGRES_PASSWORD) + .with_env("POSTGRES_DB", POSTGRES_DB) + ) + + container.start() + + registry_config = _given_registry_config_for_pg_sql(container, 2, "thread") + + yield SqlRegistry(registry_config, "project", None) + + container.stop() + + +def _given_registry_config_for_pg_sql( + container, cache_ttl_seconds=2, cache_mode="sync" +): log_string_to_wait_for = "database system is ready to accept connections" waited = wait_for_logs( container=container, @@ -148,25 +177,42 @@ def pg_registry(): container_port = container.get_exposed_port(5432) container_host = container.get_container_host_ip() - registry_config = RegistryConfig( + return RegistryConfig( registry_type="sql", + cache_ttl_seconds=cache_ttl_seconds, + cache_mode=cache_mode, # The `path` must include `+psycopg` in order for `sqlalchemy.create_engine()` # to understand that we are using psycopg3. path=f"postgresql+psycopg://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) + +@pytest.fixture(scope="function") +def mysql_registry(): + container = MySqlContainer("mysql:latest") + container.start() + + registry_config = _given_registry_config_for_mysql(container) + yield SqlRegistry(registry_config, "project", None) container.stop() -@pytest.fixture(scope="session") -def mysql_registry(): +@pytest.fixture(scope="function") +def mysql_registry_async(): container = MySqlContainer("mysql:latest") container.start() - # testing for the database to exist and ready to connect and start testing. + registry_config = _given_registry_config_for_mysql(container, 2, "thread") + + yield SqlRegistry(registry_config, "project", None) + + container.stop() + + +def _given_registry_config_for_mysql(container, cache_ttl_seconds=2, cache_mode="sync"): import sqlalchemy engine = sqlalchemy.create_engine( @@ -174,16 +220,14 @@ def mysql_registry(): ) engine.connect() - registry_config = RegistryConfig( + return RegistryConfig( registry_type="sql", path=container.get_connection_url(), + cache_ttl_seconds=cache_ttl_seconds, + cache_mode=cache_mode, sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) - yield SqlRegistry(registry_config, "project", None) - - container.stop() - @pytest.fixture(scope="session") def sqlite_registry(): @@ -269,6 +313,17 @@ def mock_remote_registry(): lazy_fixture("sqlite_registry"), ] +async_sql_fixtures = [ + pytest.param( + lazy_fixture("pg_registry_async"), + marks=pytest.mark.xdist_group(name="pg_registry_async"), + ), + pytest.param( + lazy_fixture("mysql_registry_async"), + marks=pytest.mark.xdist_group(name="mysql_registry_async"), + ), +] + @pytest.mark.integration @pytest.mark.parametrize("test_registry", all_fixtures) @@ -999,6 +1054,44 @@ def test_registry_cache(test_registry): test_registry.teardown() +@pytest.mark.integration +@pytest.mark.parametrize( + "test_registry", + async_sql_fixtures, +) +def test_registry_cache_thread_async(test_registry): + # Create Feature View + batch_source = FileSource( + name="test_source", + file_format=ParquetFormat(), + path="file://feast/*", + timestamp_field="ts_col", + created_timestamp_column="timestamp", + ) + + project = "project" + + # Register data source + test_registry.apply_data_source(batch_source, project) + registry_data_sources_cached = test_registry.list_data_sources( + project, allow_cache=True + ) + # async ttl yet to expire, so there will be a cache miss + assert len(registry_data_sources_cached) == 0 + + # Wait for cache to be refreshed + time.sleep(4) + # Now objects exist + registry_data_sources_cached = test_registry.list_data_sources( + project, allow_cache=True + ) + assert len(registry_data_sources_cached) == 1 + registry_data_source = registry_data_sources_cached[0] + assert registry_data_source == batch_source + + test_registry.teardown() + + @pytest.mark.integration @pytest.mark.parametrize( "test_registry", From 8e8c1f2ff9a77738e71542cbaab9531f321842a4 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Wed, 10 Jul 2024 09:20:41 -0400 Subject: [PATCH 13/33] feat: Add Tornike to maintainers.md (#4339) * Update maintainers.md * Update maintainers.md * Update maintainers.md --- community/maintainers.md | 1 + 1 file changed, 1 insertion(+) diff --git a/community/maintainers.md b/community/maintainers.md index 0b3d4ab648..5ccd347be0 100644 --- a/community/maintainers.md +++ b/community/maintainers.md @@ -16,6 +16,7 @@ In alphabetical order | Shuchu Han | `shuchu` | shuchu.han@gmail.com | Independent | | Willem Pienaar | `woop` | will.pienaar@gmail.com | Cleric | | Zhiling Chen | `zhilingc` | chnzhlng@gmail.com | GetGround | +| Tornike Gurgenidze | `tokoko` | togurgenidze@gmail.com | Bank of Georgia | ## Emeritus Maintainers From 71afd1c31d2a0ebf033f72aaf27eba8b9d66d4db Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Wed, 10 Jul 2024 10:10:40 -0400 Subject: [PATCH 14/33] docs: Update SUMMARY.md (#4340) Update SUMMARY.md --- docs/SUMMARY.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index a40c60d97c..5a82a190fe 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -121,7 +121,8 @@ * [\[Alpha\] Go feature server](reference/feature-servers/go-feature-server.md) * [Offline Feature Server](reference/feature-servers/offline-feature-server) * [\[Beta\] Web UI](reference/alpha-web-ui.md) -* [\[Alpha\] On demand feature view](reference/alpha-on-demand-feature-view.md) +* [\[Beta\] On demand feature view](reference/beta-on-demand-feature-view.md) +* [\[Alpha\] Vector Database](reference/alpha-vector-database.md) * [\[Alpha\] Data quality monitoring](reference/dqm.md) * [Feast CLI reference](reference/feast-cli-commands.md) * [Python API reference](http://rtd.feast.dev) From 96613c108ad3f42ca38f72c25130319eef2568c6 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Wed, 10 Jul 2024 19:19:55 +0400 Subject: [PATCH 15/33] chore: Upgrade ibis to 9.0 (#4330) upgrade ibis version Signed-off-by: tokoko --- sdk/python/feast/infra/offline_stores/ibis.py | 7 +- .../requirements/py3.10-ci-requirements.txt | 167 ++---------------- .../requirements/py3.10-requirements.txt | 46 +---- .../requirements/py3.11-ci-requirements.txt | 167 ++---------------- .../requirements/py3.11-requirements.txt | 46 +---- .../requirements/py3.9-ci-requirements.txt | 167 ++---------------- .../requirements/py3.9-requirements.txt | 46 +---- setup.py | 6 +- 8 files changed, 80 insertions(+), 572 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/ibis.py b/sdk/python/feast/infra/offline_stores/ibis.py index 6cc1606a45..dd81fd1d4e 100644 --- a/sdk/python/feast/infra/offline_stores/ibis.py +++ b/sdk/python/feast/infra/offline_stores/ibis.py @@ -335,11 +335,8 @@ def deduplicate( if created_timestamp_col: order_by_fields.append(ibis.desc(table[created_timestamp_col])) - table = ( - table.group_by(by=group_by_cols) - .order_by(order_by_fields) - .mutate(rn=ibis.row_number()) - ) + window = ibis.window(group_by=group_by_cols, order_by=order_by_fields, following=0) + table = table.mutate(rn=ibis.row_number().over(window)) return table.filter(table["rn"] == ibis.literal(0)).drop("rn") diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 0eaababc0d..e6e66ac2ee 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,7 +1,6 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.13.1 - # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -20,8 +19,6 @@ anyio==4.4.0 # jupyter-server # starlette # watchfiles -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -31,7 +28,6 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 - # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -52,9 +48,7 @@ azure-core==1.30.2 # azure-identity # azure-storage-blob azure-identity==1.17.1 - # via feast (setup.py) azure-storage-blob==12.20.0 - # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -66,9 +60,7 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.131 - # via - # feast (setup.py) - # moto + # via moto botocore==1.34.131 # via # aiobotocore @@ -77,7 +69,6 @@ botocore==1.34.131 # s3transfer build==1.2.1 # via - # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -85,7 +76,6 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 - # via feast (setup.py) certifi==2024.6.2 # via # elastic-transport @@ -108,7 +98,6 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # feast (setup.py) # dask # geomet # great-expectations @@ -118,9 +107,7 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations + # via great-expectations comm==0.2.2 # via # ipykernel @@ -129,7 +116,6 @@ coverage[toml]==7.5.4 # via pytest-cov cryptography==42.0.8 # via - # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -141,9 +127,7 @@ cryptography==42.0.8 # types-pyopenssl # types-redis dask[dataframe]==2024.6.2 - # via - # feast (setup.py) - # dask-expr + # via dask-expr dask-expr==1.1.6 # via dask db-dtypes==1.2.0 @@ -155,9 +139,7 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.18.1 - # via feast (setup.py) dill==0.3.8 - # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -167,15 +149,10 @@ docker==7.1.0 docutils==0.19 # via sphinx duckdb==0.10.3 - # via - # duckdb-engine - # ibis-framework -duckdb-engine==0.13.0 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.14.0 - # via feast (setup.py) email-validator==2.2.0 # via fastapi entrypoints==0.4 @@ -190,7 +167,6 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via feast (setup.py) fastapi-cli==0.0.4 # via fastapi fastjsonschema==2.20.0 @@ -200,7 +176,6 @@ filelock==3.15.4 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -208,16 +183,13 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via - # feast (setup.py) - # dask + # via dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.1 # via - # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -242,11 +214,8 @@ google-auth==2.30.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 - # via feast (setup.py) google-cloud-bigtable==2.24.0 - # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -255,13 +224,10 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 - # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.17.0 - # via - # feast (setup.py) - # firebase-admin + # via firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -272,19 +238,16 @@ google-resumable-media==2.7.1 # google-cloud-storage googleapis-common-protos[grpc]==1.63.2 # via - # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.16 - # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -295,27 +258,19 @@ grpcio==1.64.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 - # via feast (setup.py) grpcio-reflection==1.62.2 - # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 - # via feast (setup.py) grpcio-tools==1.62.2 - # via feast (setup.py) gunicorn==22.0.0 - # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) hazelcast-python-client==5.4.0 - # via feast (setup.py) hiredis==2.3.2 - # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -326,15 +281,11 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via - # feast (setup.py) # fastapi # jupyterlab -ibis-framework[duckdb]==8.0.0 - # via - # feast (setup.py) - # ibis-substrait -ibis-substrait==3.2.0 - # via feast (setup.py) +ibis-framework[duckdb]==9.1.0 + # via ibis-substrait +ibis-substrait==4.0.0 identify==2.5.36 # via pre-commit idna==3.7 @@ -369,7 +320,6 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via - # feast (setup.py) # altair # fastapi # great-expectations @@ -393,7 +343,6 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.22.0 # via - # feast (setup.py) # altair # great-expectations # jupyter-events @@ -439,7 +388,6 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -460,17 +408,13 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 - # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 - # via feast (setup.py) mock==2.0.0 - # via feast (setup.py) moto==4.2.14 - # via feast (setup.py) msal==1.29.0 # via # azure-identity @@ -483,16 +427,11 @@ multidict==6.0.5 # via # aiohttp # yarl -multipledispatch==1.0.0 - # via ibis-framework mypy==1.10.1 - # via - # feast (setup.py) - # sqlalchemy + # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 - # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -515,7 +454,6 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via - # feast (setup.py) # altair # dask # db-dtypes @@ -535,7 +473,6 @@ packaging==24.1 # build # dask # db-dtypes - # duckdb-engine # google-cloud-bigquery # great-expectations # gunicorn @@ -551,7 +488,6 @@ packaging==24.1 # sphinx pandas==2.2.2 # via - # feast (setup.py) # altair # dask # dask-expr @@ -577,7 +513,6 @@ pexpect==4.9.0 pip==24.1.1 # via pip-tools pip-tools==7.4.1 - # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -590,7 +525,6 @@ ply==3.11 portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 - # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.47 @@ -605,7 +539,6 @@ proto-plus==1.24.0 # google-cloud-firestore protobuf==4.25.3 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -623,11 +556,8 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel + # via ipykernel psycopg[binary, pool]==3.1.19 - # via feast (setup.py) psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.2 @@ -639,14 +569,12 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via - # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -664,19 +592,16 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.4 # via - # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via - # feast (setup.py) # ipython # nbconvert # rich @@ -687,11 +612,8 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 - # via feast (setup.py) pymysql==1.1.1 - # via feast (setup.py) pyodbc==5.1.0 - # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -703,10 +625,8 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 - # via feast (setup.py) pytest==7.4.4 # via - # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -716,21 +636,13 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) pytest-cov==5.0.0 - # via feast (setup.py) pytest-env==1.1.3 - # via feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) pytest-xdist==3.6.1 - # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -759,7 +671,6 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via - # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -773,19 +684,15 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 - # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 - # via - # feast (setup.py) - # parsimonious + # via parsimonious requests==2.32.3 # via - # feast (setup.py) # azure-core # cachecontrol # docker @@ -820,7 +727,6 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 - # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -830,7 +736,6 @@ rsa==4.9 ruamel-yaml==0.17.17 # via great-expectations ruff==0.4.10 - # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.0 @@ -847,7 +752,6 @@ setuptools==70.1.1 shellingham==1.5.4 # via typer singlestoredb==1.4.0 - # via feast (setup.py) six==1.16.0 # via # asttokens @@ -868,13 +772,11 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.11.0 - # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -888,17 +790,9 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.31 - # via - # feast (setup.py) - # duckdb-engine - # ibis-framework - # sqlalchemy-views -sqlalchemy-views==0.3.2 - # via ibis-framework -sqlglot==20.11.0 +sqlglot==25.1.0 # via ibis-framework sqlite-vec==0.0.1a10 - # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -908,21 +802,17 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 - # via feast (setup.py) tenacity==8.4.2 - # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 - # via feast (setup.py) thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via # build @@ -950,9 +840,7 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via - # feast (setup.py) - # great-expectations + # via great-expectations traitlets==5.14.3 # via # comm @@ -969,39 +857,25 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 - # via feast (setup.py) typeguard==4.3.0 - # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf types-pymysql==1.1.0.20240524 - # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via - # feast (setup.py) - # arrow + # via arrow types-pytz==2024.1.0.20240417 - # via feast (setup.py) types-pyyaml==6.0.12.20240311 - # via feast (setup.py) types-redis==4.6.0.20240425 - # via feast (setup.py) types-requests==2.30.0.0 - # via feast (setup.py) types-setuptools==70.1.0.20240627 - # via - # feast (setup.py) - # types-cffi + # via types-cffi types-tabulate==0.9.0.20240106 - # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -1040,7 +914,6 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via - # feast (setup.py) # botocore # docker # elastic-transport @@ -1052,15 +925,11 @@ urllib3==1.26.19 # rockset # testcontainers uvicorn[standard]==0.30.1 - # via - # feast (setup.py) - # fastapi + # via fastapi uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit + # via pre-commit watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 308123600c..99c9bfc3fe 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -20,22 +20,17 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) dask[dataframe]==2024.5.0 - # via - # feast (setup.py) - # dask-expr + # via dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 - # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -43,9 +38,7 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via - # feast (setup.py) - # fastapi-cli + # via fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 @@ -53,7 +46,6 @@ fsspec==2024.3.1 greenlet==3.0.3 # via sqlalchemy gunicorn==22.0.0 - # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -73,11 +65,8 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via - # feast (setup.py) - # fastapi + # via fastapi jsonschema==4.22.0 - # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -89,16 +78,13 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 - # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 - # via feast (setup.py) numpy==1.26.4 # via - # feast (setup.py) # dask # pandas # pyarrow @@ -110,29 +96,20 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via - # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf pyarrow==16.0.0 - # via - # feast (setup.py) - # dask-expr + # via dask-expr pydantic==2.7.1 - # via - # feast (setup.py) - # fastapi + # via fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via - # feast (setup.py) - # rich + # via rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -143,7 +120,6 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via - # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -151,7 +127,6 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 - # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -167,15 +142,11 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 - # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 - # via feast (setup.py) tenacity==8.3.0 - # via feast (setup.py) toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -183,9 +154,7 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 - # via feast (setup.py) typeguard==4.2.1 - # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -209,7 +178,6 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via - # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index d0663a2bea..fa1f24a586 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,7 +1,6 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.13.1 - # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -20,8 +19,6 @@ anyio==4.4.0 # jupyter-server # starlette # watchfiles -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -31,7 +28,6 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 - # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -48,9 +44,7 @@ azure-core==1.30.2 # azure-identity # azure-storage-blob azure-identity==1.17.1 - # via feast (setup.py) azure-storage-blob==12.20.0 - # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -62,9 +56,7 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.131 - # via - # feast (setup.py) - # moto + # via moto botocore==1.34.131 # via # aiobotocore @@ -73,7 +65,6 @@ botocore==1.34.131 # s3transfer build==1.2.1 # via - # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -81,7 +72,6 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 - # via feast (setup.py) certifi==2024.6.2 # via # elastic-transport @@ -104,7 +94,6 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # feast (setup.py) # dask # geomet # great-expectations @@ -114,9 +103,7 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations + # via great-expectations comm==0.2.2 # via # ipykernel @@ -125,7 +112,6 @@ coverage[toml]==7.5.4 # via pytest-cov cryptography==42.0.8 # via - # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -137,9 +123,7 @@ cryptography==42.0.8 # types-pyopenssl # types-redis dask[dataframe]==2024.6.2 - # via - # feast (setup.py) - # dask-expr + # via dask-expr dask-expr==1.1.6 # via dask db-dtypes==1.2.0 @@ -151,9 +135,7 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.18.1 - # via feast (setup.py) dill==0.3.8 - # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -163,15 +145,10 @@ docker==7.1.0 docutils==0.19 # via sphinx duckdb==0.10.3 - # via - # duckdb-engine - # ibis-framework -duckdb-engine==0.13.0 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.14.0 - # via feast (setup.py) email-validator==2.2.0 # via fastapi entrypoints==0.4 @@ -181,7 +158,6 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via feast (setup.py) fastapi-cli==0.0.4 # via fastapi fastjsonschema==2.20.0 @@ -191,7 +167,6 @@ filelock==3.15.4 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -199,16 +174,13 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via - # feast (setup.py) - # dask + # via dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.1 # via - # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -233,11 +205,8 @@ google-auth==2.30.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 - # via feast (setup.py) google-cloud-bigtable==2.24.0 - # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -246,13 +215,10 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 - # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.17.0 - # via - # feast (setup.py) - # firebase-admin + # via firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -263,19 +229,16 @@ google-resumable-media==2.7.1 # google-cloud-storage googleapis-common-protos[grpc]==1.63.2 # via - # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.16 - # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -286,27 +249,19 @@ grpcio==1.64.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 - # via feast (setup.py) grpcio-reflection==1.62.2 - # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 - # via feast (setup.py) grpcio-tools==1.62.2 - # via feast (setup.py) gunicorn==22.0.0 - # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) hazelcast-python-client==5.4.0 - # via feast (setup.py) hiredis==2.3.2 - # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -317,15 +272,11 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via - # feast (setup.py) # fastapi # jupyterlab -ibis-framework[duckdb]==8.0.0 - # via - # feast (setup.py) - # ibis-substrait -ibis-substrait==3.2.0 - # via feast (setup.py) +ibis-framework[duckdb]==9.1.0 + # via ibis-substrait +ibis-substrait==4.0.0 identify==2.5.36 # via pre-commit idna==3.7 @@ -360,7 +311,6 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via - # feast (setup.py) # altair # fastapi # great-expectations @@ -384,7 +334,6 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.22.0 # via - # feast (setup.py) # altair # great-expectations # jupyter-events @@ -430,7 +379,6 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -451,17 +399,13 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 - # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 - # via feast (setup.py) mock==2.0.0 - # via feast (setup.py) moto==4.2.14 - # via feast (setup.py) msal==1.29.0 # via # azure-identity @@ -474,16 +418,11 @@ multidict==6.0.5 # via # aiohttp # yarl -multipledispatch==1.0.0 - # via ibis-framework mypy==1.10.1 - # via - # feast (setup.py) - # sqlalchemy + # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 - # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -506,7 +445,6 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via - # feast (setup.py) # altair # dask # db-dtypes @@ -526,7 +464,6 @@ packaging==24.1 # build # dask # db-dtypes - # duckdb-engine # google-cloud-bigquery # great-expectations # gunicorn @@ -542,7 +479,6 @@ packaging==24.1 # sphinx pandas==2.2.2 # via - # feast (setup.py) # altair # dask # dask-expr @@ -568,7 +504,6 @@ pexpect==4.9.0 pip==24.1.1 # via pip-tools pip-tools==7.4.1 - # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -581,7 +516,6 @@ ply==3.11 portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 - # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.47 @@ -596,7 +530,6 @@ proto-plus==1.24.0 # google-cloud-firestore protobuf==4.25.3 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -614,11 +547,8 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel + # via ipykernel psycopg[binary, pool]==3.1.19 - # via feast (setup.py) psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.2 @@ -630,14 +560,12 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via - # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -655,19 +583,16 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.4 # via - # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via - # feast (setup.py) # ipython # nbconvert # rich @@ -678,11 +603,8 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 - # via feast (setup.py) pymysql==1.1.1 - # via feast (setup.py) pyodbc==5.1.0 - # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -694,10 +616,8 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 - # via feast (setup.py) pytest==7.4.4 # via - # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -707,21 +627,13 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) pytest-cov==5.0.0 - # via feast (setup.py) pytest-env==1.1.3 - # via feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) pytest-xdist==3.6.1 - # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -750,7 +662,6 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via - # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -764,19 +675,15 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 - # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 - # via - # feast (setup.py) - # parsimonious + # via parsimonious requests==2.32.3 # via - # feast (setup.py) # azure-core # cachecontrol # docker @@ -811,7 +718,6 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 - # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -821,7 +727,6 @@ rsa==4.9 ruamel-yaml==0.17.17 # via great-expectations ruff==0.4.10 - # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.0 @@ -838,7 +743,6 @@ setuptools==70.1.1 shellingham==1.5.4 # via typer singlestoredb==1.4.0 - # via feast (setup.py) six==1.16.0 # via # asttokens @@ -859,13 +763,11 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.11.0 - # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -879,17 +781,9 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.31 - # via - # feast (setup.py) - # duckdb-engine - # ibis-framework - # sqlalchemy-views -sqlalchemy-views==0.3.2 - # via ibis-framework -sqlglot==20.11.0 +sqlglot==25.1.0 # via ibis-framework sqlite-vec==0.0.1a10 - # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -899,21 +793,17 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 - # via feast (setup.py) tenacity==8.4.2 - # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 - # via feast (setup.py) thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 - # via feast (setup.py) tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 @@ -931,9 +821,7 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via - # feast (setup.py) - # great-expectations + # via great-expectations traitlets==5.14.3 # via # comm @@ -950,39 +838,25 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 - # via feast (setup.py) typeguard==4.3.0 - # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf types-pymysql==1.1.0.20240524 - # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via - # feast (setup.py) - # arrow + # via arrow types-pytz==2024.1.0.20240417 - # via feast (setup.py) types-pyyaml==6.0.12.20240311 - # via feast (setup.py) types-redis==4.6.0.20240425 - # via feast (setup.py) types-requests==2.30.0.0 - # via feast (setup.py) types-setuptools==70.1.0.20240627 - # via - # feast (setup.py) - # types-cffi + # via types-cffi types-tabulate==0.9.0.20240106 - # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -1018,7 +892,6 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via - # feast (setup.py) # botocore # docker # elastic-transport @@ -1030,15 +903,11 @@ urllib3==1.26.19 # rockset # testcontainers uvicorn[standard]==0.30.1 - # via - # feast (setup.py) - # fastapi + # via fastapi uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit + # via pre-commit watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index f21afdb5b1..c34b610d14 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -20,30 +20,23 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) dask[dataframe]==2024.5.0 - # via - # feast (setup.py) - # dask-expr + # via dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 - # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi fastapi==0.111.0 - # via - # feast (setup.py) - # fastapi-cli + # via fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 @@ -51,7 +44,6 @@ fsspec==2024.3.1 greenlet==3.0.3 # via sqlalchemy gunicorn==22.0.0 - # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -71,11 +63,8 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via - # feast (setup.py) - # fastapi + # via fastapi jsonschema==4.22.0 - # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -87,16 +76,13 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 - # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 - # via feast (setup.py) numpy==1.26.4 # via - # feast (setup.py) # dask # pandas # pyarrow @@ -108,29 +94,20 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via - # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf pyarrow==16.0.0 - # via - # feast (setup.py) - # dask-expr + # via dask-expr pydantic==2.7.1 - # via - # feast (setup.py) - # fastapi + # via fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via - # feast (setup.py) - # rich + # via rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -141,7 +118,6 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via - # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -149,7 +125,6 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 - # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -165,23 +140,17 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 - # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 - # via feast (setup.py) tenacity==8.3.0 - # via feast (setup.py) toml==0.10.2 - # via feast (setup.py) toolz==0.12.1 # via # dask # partd tqdm==4.66.4 - # via feast (setup.py) typeguard==4.2.1 - # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -203,7 +172,6 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via - # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index f09c666f42..ba4c6c989d 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,7 +1,6 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.13.1 - # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -20,8 +19,6 @@ anyio==4.4.0 # jupyter-server # starlette # watchfiles -appnope==0.1.4 - # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -31,7 +28,6 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 - # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -52,9 +48,7 @@ azure-core==1.30.2 # azure-identity # azure-storage-blob azure-identity==1.17.1 - # via feast (setup.py) azure-storage-blob==12.20.0 - # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -66,9 +60,7 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.131 - # via - # feast (setup.py) - # moto + # via moto botocore==1.34.131 # via # aiobotocore @@ -77,7 +69,6 @@ botocore==1.34.131 # s3transfer build==1.2.1 # via - # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -85,7 +76,6 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 - # via feast (setup.py) certifi==2024.6.2 # via # elastic-transport @@ -108,7 +98,6 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via - # feast (setup.py) # dask # geomet # great-expectations @@ -118,9 +107,7 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations + # via great-expectations comm==0.2.2 # via # ipykernel @@ -129,7 +116,6 @@ coverage[toml]==7.5.4 # via pytest-cov cryptography==42.0.8 # via - # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -141,9 +127,7 @@ cryptography==42.0.8 # types-pyopenssl # types-redis dask[dataframe]==2024.6.2 - # via - # feast (setup.py) - # dask-expr + # via dask-expr dask-expr==1.1.6 # via dask db-dtypes==1.2.0 @@ -155,9 +139,7 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.18.1 - # via feast (setup.py) dill==0.3.8 - # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -167,15 +149,10 @@ docker==7.1.0 docutils==0.19 # via sphinx duckdb==0.10.3 - # via - # duckdb-engine - # ibis-framework -duckdb-engine==0.13.0 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.14.0 - # via feast (setup.py) email-validator==2.2.0 # via fastapi entrypoints==0.4 @@ -190,7 +167,6 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 - # via feast (setup.py) fastapi-cli==0.0.4 # via fastapi fastjsonschema==2.20.0 @@ -200,7 +176,6 @@ filelock==3.15.4 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 - # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -208,16 +183,13 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via - # feast (setup.py) - # dask + # via dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.1 # via - # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -242,11 +214,8 @@ google-auth==2.30.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 - # via feast (setup.py) google-cloud-bigtable==2.24.0 - # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -255,13 +224,10 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 - # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.17.0 - # via - # feast (setup.py) - # firebase-admin + # via firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -272,19 +238,16 @@ google-resumable-media==2.7.1 # google-cloud-storage googleapis-common-protos[grpc]==1.63.2 # via - # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.16 - # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -295,27 +258,19 @@ grpcio==1.64.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 - # via feast (setup.py) grpcio-reflection==1.62.2 - # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 - # via feast (setup.py) grpcio-tools==1.62.2 - # via feast (setup.py) gunicorn==22.0.0 - # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 - # via feast (setup.py) hazelcast-python-client==5.4.0 - # via feast (setup.py) hiredis==2.3.2 - # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -326,15 +281,11 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via - # feast (setup.py) # fastapi # jupyterlab -ibis-framework[duckdb]==8.0.0 - # via - # feast (setup.py) - # ibis-substrait -ibis-substrait==3.2.0 - # via feast (setup.py) +ibis-framework[duckdb]==9.0.0 + # via ibis-substrait +ibis-substrait==4.0.0 identify==2.5.36 # via pre-commit idna==3.7 @@ -378,7 +329,6 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via - # feast (setup.py) # altair # fastapi # great-expectations @@ -402,7 +352,6 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.22.0 # via - # feast (setup.py) # altair # great-expectations # jupyter-events @@ -448,7 +397,6 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 - # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -469,17 +417,13 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 - # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 - # via feast (setup.py) mock==2.0.0 - # via feast (setup.py) moto==4.2.14 - # via feast (setup.py) msal==1.29.0 # via # azure-identity @@ -492,16 +436,11 @@ multidict==6.0.5 # via # aiohttp # yarl -multipledispatch==1.0.0 - # via ibis-framework mypy==1.10.1 - # via - # feast (setup.py) - # sqlalchemy + # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 - # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -524,7 +463,6 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via - # feast (setup.py) # altair # dask # db-dtypes @@ -544,7 +482,6 @@ packaging==24.1 # build # dask # db-dtypes - # duckdb-engine # google-cloud-bigquery # great-expectations # gunicorn @@ -560,7 +497,6 @@ packaging==24.1 # sphinx pandas==2.2.2 # via - # feast (setup.py) # altair # dask # dask-expr @@ -586,7 +522,6 @@ pexpect==4.9.0 pip==24.1.1 # via pip-tools pip-tools==7.4.1 - # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -599,7 +534,6 @@ ply==3.11 portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 - # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.47 @@ -614,7 +548,6 @@ proto-plus==1.24.0 # google-cloud-firestore protobuf==4.25.3 # via - # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -632,11 +565,8 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel + # via ipykernel psycopg[binary, pool]==3.1.18 - # via feast (setup.py) psycopg-binary==3.1.18 # via psycopg psycopg-pool==3.2.2 @@ -648,14 +578,12 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 - # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via - # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -673,19 +601,16 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 - # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.4 # via - # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via - # feast (setup.py) # ipython # nbconvert # rich @@ -696,11 +621,8 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 - # via feast (setup.py) pymysql==1.1.1 - # via feast (setup.py) pyodbc==5.1.0 - # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -712,10 +634,8 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 - # via feast (setup.py) pytest==7.4.4 # via - # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -725,21 +645,13 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 - # via feast (setup.py) pytest-cov==5.0.0 - # via feast (setup.py) pytest-env==1.1.3 - # via feast (setup.py) pytest-lazy-fixture==0.6.3 - # via feast (setup.py) pytest-mock==1.10.4 - # via feast (setup.py) pytest-ordering==0.6 - # via feast (setup.py) pytest-timeout==1.4.2 - # via feast (setup.py) pytest-xdist==3.6.1 - # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -768,7 +680,6 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via - # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -782,19 +693,15 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 - # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 - # via - # feast (setup.py) - # parsimonious + # via parsimonious requests==2.32.3 # via - # feast (setup.py) # azure-core # cachecontrol # docker @@ -829,7 +736,6 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 - # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -841,7 +747,6 @@ ruamel-yaml==0.17.17 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.4.10 - # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.13.1 @@ -858,7 +763,6 @@ setuptools==70.1.1 shellingham==1.5.4 # via typer singlestoredb==1.4.0 - # via feast (setup.py) six==1.16.0 # via # asttokens @@ -879,13 +783,11 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.11.0 - # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -899,17 +801,9 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.31 - # via - # feast (setup.py) - # duckdb-engine - # ibis-framework - # sqlalchemy-views -sqlalchemy-views==0.3.2 - # via ibis-framework -sqlglot==20.11.0 +sqlglot==23.12.2 # via ibis-framework sqlite-vec==0.0.1a10 - # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -919,21 +813,17 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 - # via feast (setup.py) tenacity==8.4.2 - # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 - # via feast (setup.py) thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via # build @@ -961,9 +851,7 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via - # feast (setup.py) - # great-expectations + # via great-expectations traitlets==5.14.3 # via # comm @@ -980,39 +868,25 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 - # via feast (setup.py) typeguard==4.3.0 - # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf types-pymysql==1.1.0.20240524 - # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via - # feast (setup.py) - # arrow + # via arrow types-pytz==2024.1.0.20240417 - # via feast (setup.py) types-pyyaml==6.0.12.20240311 - # via feast (setup.py) types-redis==4.6.0.20240425 - # via feast (setup.py) types-requests==2.30.0.0 - # via feast (setup.py) types-setuptools==70.1.0.20240627 - # via - # feast (setup.py) - # types-cffi + # via types-cffi types-tabulate==0.9.0.20240106 - # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -1053,7 +927,6 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via - # feast (setup.py) # botocore # docker # elastic-transport @@ -1066,15 +939,11 @@ urllib3==1.26.19 # snowflake-connector-python # testcontainers uvicorn[standard]==0.30.1 - # via - # feast (setup.py) - # fastapi + # via fastapi uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit + # via pre-commit watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 52ff8a0f4f..149a96626e 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -20,22 +20,17 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) dask[dataframe]==2024.5.0 - # via - # feast (setup.py) - # dask-expr + # via dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 - # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -43,9 +38,7 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via - # feast (setup.py) - # fastapi-cli + # via fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 @@ -53,7 +46,6 @@ fsspec==2024.3.1 greenlet==3.0.3 # via sqlalchemy gunicorn==22.0.0 - # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -75,11 +67,8 @@ importlib-metadata==7.1.0 # dask # typeguard jinja2==3.1.4 - # via - # feast (setup.py) - # fastapi + # via fastapi jsonschema==4.22.0 - # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -91,16 +80,13 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 - # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 - # via feast (setup.py) numpy==1.26.4 # via - # feast (setup.py) # dask # pandas # pyarrow @@ -112,29 +98,20 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via - # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf + # via mypy-protobuf pyarrow==16.0.0 - # via - # feast (setup.py) - # dask-expr + # via dask-expr pydantic==2.7.1 - # via - # feast (setup.py) - # fastapi + # via fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via - # feast (setup.py) - # rich + # via rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -145,7 +122,6 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via - # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -153,7 +129,6 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 - # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -169,15 +144,11 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 - # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 - # via feast (setup.py) tenacity==8.3.0 - # via feast (setup.py) toml==0.10.2 - # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -185,9 +156,7 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 - # via feast (setup.py) typeguard==4.2.1 - # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -212,7 +181,6 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via - # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/setup.py b/setup.py index 958e93799d..f7fa610fbd 100644 --- a/setup.py +++ b/setup.py @@ -138,8 +138,8 @@ ] IBIS_REQUIRED = [ - "ibis-framework>=8.0.0,<9", - "ibis-substrait<=3.2.0", + "ibis-framework>=9.0.0,<10", + "ibis-substrait>=4.0.0", ] GRPCIO_REQUIRED = [ @@ -149,7 +149,7 @@ "grpcio-health-checking>=1.56.2,<2", ] -DUCKDB_REQUIRED = ["ibis-framework[duckdb]>=8.0.0,<9"] +DUCKDB_REQUIRED = ["ibis-framework[duckdb]>=9.0.0,<10"] DELTA_REQUIRED = ["deltalake"] From a639d617c047030f75c6950e9bfa6e5cfe63daaa Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Wed, 10 Jul 2024 08:22:13 -0700 Subject: [PATCH 16/33] fix: Update dask version to support pandas 1.x (#4326) * update dask version to support pandas 1.x Signed-off-by: cmuhao * update dask version to support pandas 1.x Signed-off-by: cmuhao * update dask version to support pandas 1.x Signed-off-by: cmuhao --------- Signed-off-by: cmuhao --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7fa610fbd..a543262821 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", - "dask[dataframe]>=2024.4.2", + "dask[dataframe]>=2024.2.1", ] GCP_REQUIRED = [ From c45ff72f821404c595477e696ab4be1b888090cc Mon Sep 17 00:00:00 2001 From: Alex Mirrington <34053287+alexmirrington@users.noreply.github.com> Date: Thu, 11 Jul 2024 03:18:28 +1000 Subject: [PATCH 17/33] fix: OnDemandFeatureView type inference for array types (#4310) Fix OnDemandFeatureView type inference for array types Signed-off-by: Alex Mirrington --- .../transformation/pandas_transformation.py | 28 +- .../transformation/python_transformation.py | 27 +- .../substrait_transformation.py | 32 ++- sdk/python/feast/type_map.py | 1 + .../test_on_demand_pandas_transformation.py | 254 +++++++++++++++++- .../test_on_demand_python_transformation.py | 243 ++++++++++++++++- 6 files changed, 555 insertions(+), 30 deletions(-) diff --git a/sdk/python/feast/transformation/pandas_transformation.py b/sdk/python/feast/transformation/pandas_transformation.py index e9dab72160..41e437fb6b 100644 --- a/sdk/python/feast/transformation/pandas_transformation.py +++ b/sdk/python/feast/transformation/pandas_transformation.py @@ -40,15 +40,27 @@ def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: df = pd.DataFrame.from_dict(random_input) output_df: pd.DataFrame = self.transform(df) - return [ - Field( - name=f, - dtype=from_value_type( - python_type_to_feast_value_type(f, type_name=str(dt)) - ), + fields = [] + for feature_name, feature_type in zip(output_df.columns, output_df.dtypes): + feature_value = output_df[feature_name].tolist() + if len(feature_value) <= 0: + raise TypeError( + f"Failed to infer type for feature '{feature_name}' with value " + + f"'{feature_value}' since no items were returned by the UDF." + ) + fields.append( + Field( + name=feature_name, + dtype=from_value_type( + python_type_to_feast_value_type( + feature_name, + value=feature_value[0], + type_name=str(feature_type), + ) + ), + ) ) - for f, dt in zip(output_df.columns, output_df.dtypes) - ] + return fields def __eq__(self, other): if not isinstance(other, PandasTransformation): diff --git a/sdk/python/feast/transformation/python_transformation.py b/sdk/python/feast/transformation/python_transformation.py index 2a9c7db876..d828890b1e 100644 --- a/sdk/python/feast/transformation/python_transformation.py +++ b/sdk/python/feast/transformation/python_transformation.py @@ -40,15 +40,26 @@ def transform(self, input_dict: dict) -> dict: def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: output_dict: dict[str, list[Any]] = self.transform(random_input) - return [ - Field( - name=f, - dtype=from_value_type( - python_type_to_feast_value_type(f, type_name=type(dt[0]).__name__) - ), + fields = [] + for feature_name, feature_value in output_dict.items(): + if len(feature_value) <= 0: + raise TypeError( + f"Failed to infer type for feature '{feature_name}' with value " + + f"'{feature_value}' since no items were returned by the UDF." + ) + fields.append( + Field( + name=feature_name, + dtype=from_value_type( + python_type_to_feast_value_type( + feature_name, + value=feature_value[0], + type_name=type(feature_value[0]).__name__, + ) + ), + ) ) - for f, dt in output_dict.items() - ] + return fields def __eq__(self, other): if not isinstance(other, PythonTransformation): diff --git a/sdk/python/feast/transformation/substrait_transformation.py b/sdk/python/feast/transformation/substrait_transformation.py index 17c40cf0a1..1de60aed00 100644 --- a/sdk/python/feast/transformation/substrait_transformation.py +++ b/sdk/python/feast/transformation/substrait_transformation.py @@ -60,16 +60,28 @@ def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: df = pd.DataFrame.from_dict(random_input) output_df: pd.DataFrame = self.transform(df) - return [ - Field( - name=f, - dtype=from_value_type( - python_type_to_feast_value_type(f, type_name=str(dt)) - ), - ) - for f, dt in zip(output_df.columns, output_df.dtypes) - if f not in random_input - ] + fields = [] + for feature_name, feature_type in zip(output_df.columns, output_df.dtypes): + feature_value = output_df[feature_name].tolist() + if len(feature_value) <= 0: + raise TypeError( + f"Failed to infer type for feature '{feature_name}' with value " + + f"'{feature_value}' since no items were returned by the UDF." + ) + if feature_name not in random_input: + fields.append( + Field( + name=feature_name, + dtype=from_value_type( + python_type_to_feast_value_type( + feature_name, + value=feature_value[0], + type_name=str(feature_type), + ) + ), + ) + ) + return fields def __eq__(self, other): if not isinstance(other, SubstraitTransformation): diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index a0859f2f7a..6ba61fc8c5 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -155,6 +155,7 @@ def python_type_to_feast_value_type( "uint16": ValueType.INT32, "uint8": ValueType.INT32, "int8": ValueType.INT32, + "bool_": ValueType.BOOL, # np.bool_ "bool": ValueType.BOOL, "boolean": ValueType.BOOL, "timedelta": ValueType.UNIX_TIMESTAMP, diff --git a/sdk/python/tests/unit/test_on_demand_pandas_transformation.py b/sdk/python/tests/unit/test_on_demand_pandas_transformation.py index c5f066dd83..1a04a466fb 100644 --- a/sdk/python/tests/unit/test_on_demand_pandas_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_pandas_transformation.py @@ -1,15 +1,31 @@ import os +import re import tempfile from datetime import datetime, timedelta import pandas as pd +import pytest -from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig +from feast import ( + Entity, + FeatureStore, + FeatureView, + FileSource, + RepoConfig, + RequestSource, +) from feast.driver_test_data import create_driver_hourly_stats_df from feast.field import Field from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Float64, Int64 +from feast.types import ( + Array, + Bool, + Float32, + Float64, + Int64, + String, +) def test_pandas_transformation(): @@ -91,3 +107,237 @@ def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: assert online_response["conv_rate_plus_acc"].equals( online_response["conv_rate"] + online_response["acc_rate"] ) + + +def test_pandas_transformation_returning_all_data_types(): + with tempfile.TemporaryDirectory() as data_dir: + store = FeatureStore( + config=RepoConfig( + project="test_on_demand_python_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + # Generate test data. + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) + driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") + driver_df.to_parquet(path=driver_stats_path, allow_truncated_timestamps=True) + + driver = Entity(name="driver", join_keys=["driver_id"]) + + driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=driver_stats_path, + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=0), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + request_source = RequestSource( + name="request_source", + schema=[ + Field(name="avg_daily_trip_rank_thresholds", dtype=Array(Int64)), + Field(name="avg_daily_trip_rank_names", dtype=Array(String)), + ], + ) + + @on_demand_feature_view( + sources=[request_source, driver_stats_fv], + schema=[ + Field(name="highest_achieved_rank", dtype=String), + Field(name="avg_daily_trips_plus_one", dtype=Int64), + Field(name="conv_rate_plus_acc", dtype=Float64), + Field(name="is_highest_rank", dtype=Bool), + Field(name="achieved_ranks", dtype=Array(String)), + Field(name="trips_until_next_rank_int", dtype=Array(Int64)), + Field(name="trips_until_next_rank_float", dtype=Array(Float64)), + Field(name="achieved_ranks_mask", dtype=Array(Bool)), + ], + mode="pandas", + ) + def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_acc"] = inputs["conv_rate"] + inputs["acc_rate"] + df["avg_daily_trips_plus_one"] = inputs["avg_daily_trips"] + 1 + + df["trips_until_next_rank_int"] = inputs[ + ["avg_daily_trips", "avg_daily_trip_rank_thresholds"] + ].apply( + lambda x: [max(threshold - x.iloc[0], 0) for threshold in x.iloc[1]], + axis=1, + ) + df["trips_until_next_rank_float"] = df["trips_until_next_rank_int"].map( + lambda values: [float(value) for value in values] + ) + df["achieved_ranks_mask"] = df["trips_until_next_rank_int"].map( + lambda values: [value <= 0 for value in values] + ) + + temp = pd.concat( + [df[["achieved_ranks_mask"]], inputs[["avg_daily_trip_rank_names"]]], + axis=1, + ) + df["achieved_ranks"] = temp.apply( + lambda x: [ + rank if achieved else "Locked" + for achieved, rank in zip(x.iloc[0], x.iloc[1]) + ], + axis=1, + ) + df["highest_achieved_rank"] = ( + df["achieved_ranks"] + .map( + lambda ranks: str( + ([rank for rank in ranks if rank != "Locked"][-1:] or ["None"])[ + 0 + ] + ) + ) + .astype("string") + ) + df["is_highest_rank"] = df["achieved_ranks"].map( + lambda ranks: ranks[-1] != "Locked" + ) + return df + + store.apply([driver, driver_stats_source, driver_stats_fv, pandas_view]) + + entity_rows = [ + { + "driver_id": 1001, + "avg_daily_trip_rank_thresholds": [100, 250, 500, 1000], + "avg_daily_trip_rank_names": ["Bronze", "Silver", "Gold", "Platinum"], + } + ] + store.write_to_online_store( + feature_view_name="driver_hourly_stats", df=driver_df + ) + + online_response = store.get_online_features( + entity_rows=entity_rows, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "pandas_view:avg_daily_trips_plus_one", + "pandas_view:conv_rate_plus_acc", + "pandas_view:trips_until_next_rank_int", + "pandas_view:trips_until_next_rank_float", + "pandas_view:achieved_ranks_mask", + "pandas_view:achieved_ranks", + "pandas_view:highest_achieved_rank", + "pandas_view:is_highest_rank", + ], + ).to_df() + # We use to_df here to ensure we use the pandas backend, but convert to a dict for comparisons + result = online_response.to_dict(orient="records")[0] + + # Type assertions + # Materialized view + assert type(result["conv_rate"]) == float + assert type(result["acc_rate"]) == float + assert type(result["avg_daily_trips"]) == int + # On-demand view + assert type(result["avg_daily_trips_plus_one"]) == int + assert type(result["conv_rate_plus_acc"]) == float + assert type(result["highest_achieved_rank"]) == str + assert type(result["is_highest_rank"]) == bool + + assert type(result["trips_until_next_rank_int"]) == list + assert all([type(e) == int for e in result["trips_until_next_rank_int"]]) + + assert type(result["trips_until_next_rank_float"]) == list + assert all([type(e) == float for e in result["trips_until_next_rank_float"]]) + + assert type(result["achieved_ranks"]) == list + assert all([type(e) == str for e in result["achieved_ranks"]]) + + assert type(result["achieved_ranks_mask"]) == list + assert all([type(e) == bool for e in result["achieved_ranks_mask"]]) + + # Value assertions + expected_trips_until_next_rank = [ + max(threshold - result["avg_daily_trips"], 0) + for threshold in entity_rows[0]["avg_daily_trip_rank_thresholds"] + ] + expected_mask = [value <= 0 for value in expected_trips_until_next_rank] + expected_ranks = [ + rank if achieved else "Locked" + for achieved, rank in zip( + expected_mask, entity_rows[0]["avg_daily_trip_rank_names"] + ) + ] + highest_rank = ( + [rank for rank in expected_ranks if rank != "Locked"][-1:] or ["None"] + )[0] + + assert result["conv_rate_plus_acc"] == result["conv_rate"] + result["acc_rate"] + assert result["avg_daily_trips_plus_one"] == result["avg_daily_trips"] + 1 + assert result["highest_achieved_rank"] == highest_rank + assert result["is_highest_rank"] == (expected_ranks[-1] != "Locked") + + assert result["trips_until_next_rank_int"] == expected_trips_until_next_rank + assert result["trips_until_next_rank_float"] == [ + float(value) for value in expected_trips_until_next_rank + ] + assert result["achieved_ranks_mask"] == expected_mask + assert result["achieved_ranks"] == expected_ranks + + +def test_invalid_pandas_transformation_raises_type_error_on_apply(): + with tempfile.TemporaryDirectory() as data_dir: + store = FeatureStore( + config=RepoConfig( + project="test_on_demand_python_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + request_source = RequestSource( + name="request_source", + schema=[ + Field(name="driver_name", dtype=String), + ], + ) + + @on_demand_feature_view( + sources=[request_source], + schema=[Field(name="driver_name_lower", dtype=String)], + mode="pandas", + ) + def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: + return pd.DataFrame({"driver_name_lower": []}) + + with pytest.raises( + TypeError, + match=re.escape( + "Failed to infer type for feature 'driver_name_lower' with value '[]' since no items were returned by the UDF." + ), + ): + store.apply([request_source, pandas_view]) diff --git a/sdk/python/tests/unit/test_on_demand_python_transformation.py b/sdk/python/tests/unit/test_on_demand_python_transformation.py index 72e9b53a10..c5bd68d6a8 100644 --- a/sdk/python/tests/unit/test_on_demand_python_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_python_transformation.py @@ -1,4 +1,5 @@ import os +import re import tempfile import unittest from datetime import datetime, timedelta @@ -7,12 +8,19 @@ import pandas as pd import pytest -from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig +from feast import ( + Entity, + FeatureStore, + FeatureView, + FileSource, + RepoConfig, + RequestSource, +) from feast.driver_test_data import create_driver_hourly_stats_df from feast.field import Field from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Float64, Int64 +from feast.types import Array, Bool, Float32, Float64, Int64, String class TestOnDemandPythonTransformation(unittest.TestCase): @@ -248,3 +256,234 @@ def test_python_docs_demo(self): + online_python_response["acc_rate"][0] == online_python_response["conv_rate_plus_val2_python"][0] ) + + +class TestOnDemandPythonTransformationAllDataTypes(unittest.TestCase): + def setUp(self): + with tempfile.TemporaryDirectory() as data_dir: + self.store = FeatureStore( + config=RepoConfig( + project="test_on_demand_python_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + # Generate test data. + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df( + driver_entities, start_date, end_date + ) + driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") + driver_df.to_parquet( + path=driver_stats_path, allow_truncated_timestamps=True + ) + + driver = Entity(name="driver", join_keys=["driver_id"]) + + driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=driver_stats_path, + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=0), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + request_source = RequestSource( + name="request_source", + schema=[ + Field(name="avg_daily_trip_rank_thresholds", dtype=Array(Int64)), + Field(name="avg_daily_trip_rank_names", dtype=Array(String)), + ], + ) + + @on_demand_feature_view( + sources=[request_source, driver_stats_fv], + schema=[ + Field(name="highest_achieved_rank", dtype=String), + Field(name="avg_daily_trips_plus_one", dtype=Int64), + Field(name="conv_rate_plus_acc", dtype=Float64), + Field(name="is_highest_rank", dtype=Bool), + Field(name="achieved_ranks", dtype=Array(String)), + Field(name="trips_until_next_rank_int", dtype=Array(Int64)), + Field(name="trips_until_next_rank_float", dtype=Array(Float64)), + Field(name="achieved_ranks_mask", dtype=Array(Bool)), + ], + mode="python", + ) + def python_view(inputs: dict[str, Any]) -> dict[str, Any]: + output = {} + trips_until_next_rank = [ + [max(threshold - row[1], 0) for threshold in row[0]] + for row in zip( + inputs["avg_daily_trip_rank_thresholds"], + inputs["avg_daily_trips"], + ) + ] + mask = [[value <= 0 for value in row] for row in trips_until_next_rank] + ranks = [ + [rank if mask else "Locked" for mask, rank in zip(*row)] + for row in zip(mask, inputs["avg_daily_trip_rank_names"]) + ] + highest_rank = [ + ([rank for rank in row if rank != "Locked"][-1:] or ["None"])[0] + for row in ranks + ] + + output["conv_rate_plus_acc"] = [ + sum(row) for row in zip(inputs["conv_rate"], inputs["acc_rate"]) + ] + output["avg_daily_trips_plus_one"] = [ + row + 1 for row in inputs["avg_daily_trips"] + ] + output["highest_achieved_rank"] = highest_rank + output["is_highest_rank"] = [row[-1] != "Locked" for row in ranks] + + output["trips_until_next_rank_int"] = trips_until_next_rank + output["trips_until_next_rank_float"] = [ + [float(value) for value in row] for row in trips_until_next_rank + ] + output["achieved_ranks_mask"] = mask + output["achieved_ranks"] = ranks + return output + + self.store.apply( + [driver, driver_stats_source, driver_stats_fv, python_view] + ) + self.store.write_to_online_store( + feature_view_name="driver_hourly_stats", df=driver_df + ) + + def test_python_transformation_returning_all_data_types(self): + entity_rows = [ + { + "driver_id": 1001, + "avg_daily_trip_rank_thresholds": [100, 250, 500, 1000], + "avg_daily_trip_rank_names": ["Bronze", "Silver", "Gold", "Platinum"], + } + ] + online_response = self.store.get_online_features( + entity_rows=entity_rows, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "python_view:avg_daily_trips_plus_one", + "python_view:conv_rate_plus_acc", + "python_view:trips_until_next_rank_int", + "python_view:trips_until_next_rank_float", + "python_view:achieved_ranks_mask", + "python_view:achieved_ranks", + "python_view:highest_achieved_rank", + "python_view:is_highest_rank", + ], + ).to_dict() + result = {name: value[0] for name, value in online_response.items()} + + # Type assertions + # Materialized view + assert type(result["conv_rate"]) == float + assert type(result["acc_rate"]) == float + assert type(result["avg_daily_trips"]) == int + # On-demand view + assert type(result["avg_daily_trips_plus_one"]) == int + assert type(result["conv_rate_plus_acc"]) == float + assert type(result["highest_achieved_rank"]) == str + assert type(result["is_highest_rank"]) == bool + + assert type(result["trips_until_next_rank_int"]) == list + assert all([type(e) == int for e in result["trips_until_next_rank_int"]]) + + assert type(result["trips_until_next_rank_float"]) == list + assert all([type(e) == float for e in result["trips_until_next_rank_float"]]) + + assert type(result["achieved_ranks"]) == list + assert all([type(e) == str for e in result["achieved_ranks"]]) + + assert type(result["achieved_ranks_mask"]) == list + assert all([type(e) == bool for e in result["achieved_ranks_mask"]]) + + # Value assertions + expected_trips_until_next_rank = [ + max(threshold - result["avg_daily_trips"], 0) + for threshold in entity_rows[0]["avg_daily_trip_rank_thresholds"] + ] + expected_mask = [value <= 0 for value in expected_trips_until_next_rank] + expected_ranks = [ + rank if achieved else "Locked" + for achieved, rank in zip( + expected_mask, entity_rows[0]["avg_daily_trip_rank_names"] + ) + ] + highest_rank = ( + [rank for rank in expected_ranks if rank != "Locked"][-1:] or ["None"] + )[0] + + assert result["conv_rate_plus_acc"] == result["conv_rate"] + result["acc_rate"] + assert result["avg_daily_trips_plus_one"] == result["avg_daily_trips"] + 1 + assert result["highest_achieved_rank"] == highest_rank + assert result["is_highest_rank"] == (expected_ranks[-1] != "Locked") + + assert result["trips_until_next_rank_int"] == expected_trips_until_next_rank + assert result["trips_until_next_rank_float"] == [ + float(value) for value in expected_trips_until_next_rank + ] + assert result["achieved_ranks_mask"] == expected_mask + assert result["achieved_ranks"] == expected_ranks + + +def test_invalid_python_transformation_raises_type_error_on_apply(): + with tempfile.TemporaryDirectory() as data_dir: + store = FeatureStore( + config=RepoConfig( + project="test_on_demand_python_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + request_source = RequestSource( + name="request_source", + schema=[ + Field(name="driver_name", dtype=String), + ], + ) + + @on_demand_feature_view( + sources=[request_source], + schema=[Field(name="driver_name_lower", dtype=String)], + mode="python", + ) + def python_view(inputs: dict[str, Any]) -> dict[str, Any]: + return {"driver_name_lower": []} + + with pytest.raises( + TypeError, + match=re.escape( + "Failed to infer type for feature 'driver_name_lower' with value '[]' since no items were returned by the UDF." + ), + ): + store.apply([request_source, python_view]) From 5c07bd80ce729c5aef90e2b07710cd06c0334b1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:19:13 +0400 Subject: [PATCH 18/33] chore: Bump zipp from 3.18.1 to 3.19.1 in /sdk/python/requirements (#4337) Bumps [zipp](https://github.com/jaraco/zipp) from 3.18.1 to 3.19.1. - [Release notes](https://github.com/jaraco/zipp/releases) - [Changelog](https://github.com/jaraco/zipp/blob/main/NEWS.rst) - [Commits](https://github.com/jaraco/zipp/compare/v3.18.1...v3.19.1) --- updated-dependencies: - dependency-name: zipp dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index e6e66ac2ee..a9ac50711a 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -962,5 +962,5 @@ xmltodict==0.13.0 # via moto yarl==1.9.4 # via aiohttp -zipp==3.19.2 +zipp==3.19.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 99c9bfc3fe..b9d913c48a 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -186,5 +186,5 @@ watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn -zipp==3.18.1 +zipp==3.19.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index fa1f24a586..7f85ceb547 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -940,5 +940,5 @@ xmltodict==0.13.0 # via moto yarl==1.9.4 # via aiohttp -zipp==3.19.2 +zipp==3.19.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index c34b610d14..2bf521cda9 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -180,5 +180,5 @@ watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn -zipp==3.18.1 +zipp==3.19.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index ba4c6c989d..bbeb7367e7 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -976,5 +976,5 @@ xmltodict==0.13.0 # via moto yarl==1.9.4 # via aiohttp -zipp==3.19.2 +zipp==3.19.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 149a96626e..9c4450cb45 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -189,5 +189,5 @@ watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn -zipp==3.18.1 +zipp==3.19.1 # via importlib-metadata From aba317cf33b1c17bd94c8724522abaf15e5683ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:27:54 -0400 Subject: [PATCH 19/33] chore: Bump braces from 3.0.2 to 3.0.3 in /sdk/python/feast/ui (#4287) --- sdk/python/feast/ui/yarn.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 005035db2d..7c01b6c1e8 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -3560,11 +3560,11 @@ brace-expansion@^2.0.1: balanced-match "^1.0.0" braces@^3.0.2, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" broadcast-channel@^3.4.1: version "3.7.0" @@ -5400,10 +5400,10 @@ filesize@^8.0.6: resolved "https://registry.yarnpkg.com/filesize/-/filesize-8.0.7.tgz#695e70d80f4e47012c132d57a059e80c6b580bd8" integrity sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ== -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" From dc363472db75c2971c7711147f824257235c6673 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:28:42 -0400 Subject: [PATCH 20/33] chore: Bump ws from 7.5.6 to 7.5.10 in /ui (#4292) --- ui/yarn.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/yarn.lock b/ui/yarn.lock index 89107de0b8..26c833fa11 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -11640,14 +11640,14 @@ write-file-atomic@^3.0.0: typedarray-to-buffer "^3.1.5" ws@^7.4.6: - version "7.5.6" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.6.tgz#e59fc509fb15ddfb65487ee9765c5a51dec5fe7b" - integrity sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA== + version "7.5.10" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" + integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== ws@^8.1.0: - version "8.4.2" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.4.2.tgz#18e749868d8439f2268368829042894b6907aa0b" - integrity sha512-Kbk4Nxyq7/ZWqr/tarI9yIt/+iNNFOjBXEWgTb4ydaNHBNGgvf2QHbS9fdfsndfjFlFwEd4Al+mw83YkaD10ZA== + version "8.17.1" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b" + integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ== xml-name-validator@^3.0.0: version "3.0.0" From 660df6ec9b1191d1ea1b673c8a4fd5845c05f717 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Fri, 12 Jul 2024 01:26:03 -0400 Subject: [PATCH 21/33] chore: Updating docs (#4346) --- docs/README.md | 8 ++++---- ui/feature_repo/features.py | 22 +++++++++++----------- ui/package.json | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/README.md b/docs/README.md index 66c7548440..eea372ded0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,10 +39,10 @@ Feast is likely **not** the right tool if you ### Feast does not _fully_ solve * **reproducible model training / model backtesting / experiment management**: Feast captures feature and model metadata, but does not version-control datasets / labels or manage train / test splits. Other tools like [DVC](https://dvc.org/), [MLflow](https://www.mlflow.org/), and [Kubeflow](https://www.kubeflow.org/) are better suited for this. -* **batch + streaming feature engineering**: Feast primarily processes already transformed feature values (though it offers experimental light-weight transformations). Users usually integrate Feast with upstream systems (e.g. existing ETL/ELT pipelines). [Tecton](http://tecton.ai/) is a more fully featured feature platform which addresses these needs. -* **native streaming feature integration:** Feast enables users to push streaming features, but does not pull from streaming sources or manage streaming pipelines. [Tecton](http://tecton.ai/) is a more fully featured feature platform which orchestrates end to end streaming pipelines. -* **feature sharing**: Feast has experimental functionality to enable discovery and cataloguing of feature metadata with a [Feast web UI (alpha)](https://docs.feast.dev/reference/alpha-web-ui). Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). [Tecton](http://tecton.ai/) also more robustly addresses these needs. -* **lineage:** Feast helps tie feature values to model versions, but is not a complete solution for capturing end-to-end lineage from raw data sources to model versions. Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). [Tecton](http://tecton.ai/) captures more end-to-end lineage by also managing feature transformations. +* **batch + streaming feature engineering**: Feast primarily processes already transformed feature values but is investing in supporting batch and streaming transformations. +* **native streaming feature integration:** Feast enables users to push streaming features, but does not pull from streaming sources or manage streaming pipelines. +* **feature sharing**: Feast has experimental functionality to enable discovery and cataloguing of feature metadata with a [Feast web UI (alpha)](https://docs.feast.dev/reference/alpha-web-ui). Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). +* **lineage:** Feast helps tie feature values to model versions, but is not a complete solution for capturing end-to-end lineage from raw data sources to model versions. Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). * **data quality / drift detection**: Feast has experimental integrations with [Great Expectations](https://greatexpectations.io/), but is not purpose built to solve data drift / data quality issues. This requires more sophisticated monitoring across data pipelines, served feature values, labels, and model versions. ## Example use cases diff --git a/ui/feature_repo/features.py b/ui/feature_repo/features.py index e02bb3de5d..40a42a9e99 100644 --- a/ui/feature_repo/features.py +++ b/ui/feature_repo/features.py @@ -11,7 +11,7 @@ name="zipcode", description="A zipcode", tags={ - "owner": "danny@tecton.ai", + "owner": "danny@feast.ai", "team": "hack week", }, ) @@ -40,7 +40,7 @@ tags={ "date_added": "2022-02-7", "experiments": "experiment-A,experiment-B,experiment-C", - "access_group": "feast-team@tecton.ai", + "access_group": "feast-team@feast.ai", }, online=True, ) @@ -62,7 +62,7 @@ tags={ "date_added": "2022-02-7", "experiments": "experiment-A,experiment-B,experiment-C", - "access_group": "feast-team@tecton.ai", + "access_group": "feast-team@feast.ai", }, online=True, ) @@ -80,7 +80,7 @@ tags={ "date_added": "2022-02-7", "experiments": "experiment-A,experiment-B,experiment-C", - "access_group": "feast-team@tecton.ai", + "access_group": "feast-team@feast.ai", }, online=True, ) @@ -89,7 +89,7 @@ name="dob_ssn", description="Date of birth and last four digits of social security number", tags={ - "owner": "tony@tecton.ai", + "owner": "tony@feast.ai", "team": "hack week", }, ) @@ -121,7 +121,7 @@ tags={ "date_added": "2022-02-6", "experiments": "experiment-A", - "access_group": "feast-team@tecton.ai", + "access_group": "feast-team@feast.ai", }, online=True, ) @@ -157,7 +157,7 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame: credit_history[["credit_card_due", "missed_payments_1y"]], zipcode_features, ], - tags={"owner": "tony@tecton.ai", "stage": "staging"}, + tags={"owner": "tony@feast.ai", "stage": "staging"}, description="Credit scoring model", ) @@ -167,7 +167,7 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame: credit_history[["mortgage_due", "credit_card_due", "missed_payments_1y"]], zipcode_features, ], - tags={"owner": "tony@tecton.ai", "stage": "prod"}, + tags={"owner": "tony@feast.ai", "stage": "prod"}, description="Credit scoring model", ) @@ -178,7 +178,7 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame: zipcode_features, transaction_gt_last_credit_card_due, ], - tags={"owner": "tony@tecton.ai", "stage": "dev"}, + tags={"owner": "tony@feast.ai", "stage": "dev"}, description="Credit scoring model", ) @@ -187,7 +187,7 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame: features=[ zipcode_features, ], - tags={"owner": "amanda@tecton.ai", "stage": "dev"}, + tags={"owner": "amanda@feast.ai", "stage": "dev"}, description="Location model", ) @@ -196,6 +196,6 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame: features=[ zipcode_money_features, ], - tags={"owner": "amanda@tecton.ai", "stage": "dev"}, + tags={"owner": "amanda@feast.ai", "stage": "dev"}, description="Location model", ) diff --git a/ui/package.json b/ui/package.json index de37f4394a..a380c65cfc 100644 --- a/ui/package.json +++ b/ui/package.json @@ -118,7 +118,7 @@ "Feature", "Store" ], - "author": "tony@tecton.ai", + "author": "tony@feast.ai", "license": "Apache-2.0", "bugs": { "url": "https://github.com/feast-dev/feast/issues" From 1ce65bc1f5528a8bfa33e36e8852f4fd198d3ce7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:11:39 +0400 Subject: [PATCH 22/33] chore: Bump certifi from 2024.2.2 to 2024.7.4 in /sdk/python/requirements (#4334) chore: Bump certifi in /sdk/python/requirements Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.2.2 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2024.02.02...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index a9ac50711a..33709a1ef0 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -76,7 +76,7 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 -certifi==2024.6.2 +certifi==2024.7.4 # via # elastic-transport # httpcore diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index b9d913c48a..0cca106863 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -11,7 +11,7 @@ attrs==23.2.0 # via # jsonschema # referencing -certifi==2024.2.2 +certifi==2024.7.4 # via # httpcore # httpx diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 7f85ceb547..09e9e8eeea 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -72,7 +72,7 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 -certifi==2024.6.2 +certifi==2024.7.4 # via # elastic-transport # httpcore diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 2bf521cda9..687e4bfe52 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -11,7 +11,7 @@ attrs==23.2.0 # via # jsonschema # referencing -certifi==2024.2.2 +certifi==2024.7.4 # via # httpcore # httpx diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index bbeb7367e7..6f5d0220bc 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -76,7 +76,7 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 -certifi==2024.6.2 +certifi==2024.7.4 # via # elastic-transport # httpcore diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 9c4450cb45..096f54ab1f 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -11,7 +11,7 @@ attrs==23.2.0 # via # jsonschema # referencing -certifi==2024.2.2 +certifi==2024.7.4 # via # httpcore # httpx From 92d17def8cdff2bebfa622a4b3846d5bdc3e58d8 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Sat, 13 Jul 2024 22:19:12 -0400 Subject: [PATCH 23/33] fix: Remove typo. (#4351) --- docs/reference/feature-servers/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/reference/feature-servers/README.md b/docs/reference/feature-servers/README.md index 124834f8a7..2ceaf5807f 100644 --- a/docs/reference/feature-servers/README.md +++ b/docs/reference/feature-servers/README.md @@ -8,7 +8,6 @@ Feast users can choose to retrieve features from a feature server, as opposed to {% content-ref url="go-feature-server.md" %} [go-feature-server.md](go-feature-server.md) -======= {% endcontent-ref %} {% content-ref url="offline-feature-server.md" %} From b9696efb128b9591ca5b2a41e7a9a26e196ebac4 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Mon, 15 Jul 2024 02:47:01 +0400 Subject: [PATCH 24/33] chore: Rename FileOfflineStore to DaskOfflineStore (#4349) rename file offline store to dask Signed-off-by: tokoko --- docs/SUMMARY.md | 4 +-- .../offline-stores/{file.md => dask.md} | 17 ++++++----- docs/reference/offline-stores/overview.md | 6 ++-- .../docs/source/feast.infra.contrib.rst | 8 ------ .../source/feast.infra.feature_servers.rst | 2 -- .../source/feast.infra.offline_stores.rst | 12 ++++---- .../source/feast.infra.registry.contrib.rst | 1 - sdk/python/docs/source/feast.infra.rst | 24 ---------------- .../infra/offline_stores/{file.py => dask.py} | 28 +++++++++---------- sdk/python/feast/repo_config.py | 7 +++-- .../universal/data_sources/file.py | 6 ++-- .../offline_stores/test_offline_store.py | 8 +++--- .../test_dynamodb_online_store.py | 4 +-- 13 files changed, 46 insertions(+), 81 deletions(-) rename docs/reference/offline-stores/{file.md => dask.md} (87%) rename sdk/python/feast/infra/offline_stores/{file.py => dask.py} (97%) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 5a82a190fe..1173a693ef 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -76,7 +76,7 @@ * [Azure Synapse + Azure SQL (contrib)](reference/data-sources/mssql.md) * [Offline stores](reference/offline-stores/README.md) * [Overview](reference/offline-stores/overview.md) - * [File](reference/offline-stores/file.md) + * [Dask](reference/offline-stores/dask.md) * [Snowflake](reference/offline-stores/snowflake.md) * [BigQuery](reference/offline-stores/bigquery.md) * [Redshift](reference/offline-stores/redshift.md) @@ -119,7 +119,7 @@ * [Feature servers](reference/feature-servers/README.md) * [Python feature server](reference/feature-servers/python-feature-server.md) * [\[Alpha\] Go feature server](reference/feature-servers/go-feature-server.md) - * [Offline Feature Server](reference/feature-servers/offline-feature-server) + * [Offline Feature Server](reference/feature-servers/offline-feature-server.md) * [\[Beta\] Web UI](reference/alpha-web-ui.md) * [\[Beta\] On demand feature view](reference/beta-on-demand-feature-view.md) * [\[Alpha\] Vector Database](reference/alpha-vector-database.md) diff --git a/docs/reference/offline-stores/file.md b/docs/reference/offline-stores/dask.md similarity index 87% rename from docs/reference/offline-stores/file.md rename to docs/reference/offline-stores/dask.md index 4b76d9af90..d8698ba544 100644 --- a/docs/reference/offline-stores/file.md +++ b/docs/reference/offline-stores/dask.md @@ -1,9 +1,8 @@ -# File offline store +# Dask offline store ## Description -The file offline store provides support for reading [FileSources](../data-sources/file.md). -It uses Dask as the compute engine. +The Dask offline store provides support for reading [FileSources](../data-sources/file.md). {% hint style="warning" %} All data is downloaded and joined using Python and therefore may not scale to production workloads. @@ -17,18 +16,18 @@ project: my_feature_repo registry: data/registry.db provider: local offline_store: - type: file + type: dask ``` {% endcode %} -The full set of configuration options is available in [FileOfflineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.offline_stores.file.FileOfflineStoreConfig). +The full set of configuration options is available in [DaskOfflineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.offline_stores.dask.DaskOfflineStoreConfig). ## Functionality Matrix The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). -Below is a matrix indicating which functionality is supported by the file offline store. +Below is a matrix indicating which functionality is supported by the dask offline store. -| | File | +| | Dask | | :-------------------------------- | :-- | | `get_historical_features` (point-in-time correct join) | yes | | `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | @@ -36,9 +35,9 @@ Below is a matrix indicating which functionality is supported by the file offlin | `offline_write_batch` (persist dataframes to offline store) | yes | | `write_logged_features` (persist logged features to offline store) | yes | -Below is a matrix indicating which functionality is supported by `FileRetrievalJob`. +Below is a matrix indicating which functionality is supported by `DaskRetrievalJob`. -| | File | +| | Dask | | --------------------------------- | --- | | export to dataframe | yes | | export to arrow table | yes | diff --git a/docs/reference/offline-stores/overview.md b/docs/reference/offline-stores/overview.md index 4d7681e38c..182eac6586 100644 --- a/docs/reference/offline-stores/overview.md +++ b/docs/reference/offline-stores/overview.md @@ -25,13 +25,13 @@ The first three of these methods all return a `RetrievalJob` specific to an offl ## Functionality Matrix -There are currently four core offline store implementations: `FileOfflineStore`, `BigQueryOfflineStore`, `SnowflakeOfflineStore`, and `RedshiftOfflineStore`. +There are currently four core offline store implementations: `DaskOfflineStore`, `BigQueryOfflineStore`, `SnowflakeOfflineStore`, and `RedshiftOfflineStore`. There are several additional implementations contributed by the Feast community (`PostgreSQLOfflineStore`, `SparkOfflineStore`, and `TrinoOfflineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. Details for each specific offline store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). Below is a matrix indicating which offline stores support which methods. -| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | +| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | | :-------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | | `get_historical_features` | yes | yes | yes | yes | yes | yes | yes | | `pull_latest_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes | @@ -42,7 +42,7 @@ Below is a matrix indicating which offline stores support which methods. Below is a matrix indicating which `RetrievalJob`s support what functionality. -| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | DuckDB | +| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | DuckDB | | --------------------------------- | --- | --- | --- | --- | --- | --- | --- | --- | | export to dataframe | yes | yes | yes | yes | yes | yes | yes | yes | | export to arrow table | yes | yes | yes | yes | yes | yes | yes | yes | diff --git a/sdk/python/docs/source/feast.infra.contrib.rst b/sdk/python/docs/source/feast.infra.contrib.rst index 7b2fa3cc9c..1f46ff0abf 100644 --- a/sdk/python/docs/source/feast.infra.contrib.rst +++ b/sdk/python/docs/source/feast.infra.contrib.rst @@ -4,14 +4,6 @@ feast.infra.contrib package Submodules ---------- -feast.infra.contrib.azure\_provider module ------------------------------------------- - -.. automodule:: feast.infra.contrib.azure_provider - :members: - :undoc-members: - :show-inheritance: - feast.infra.contrib.grpc\_server module --------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.feature_servers.rst b/sdk/python/docs/source/feast.infra.feature_servers.rst index 334b585905..ca5203504d 100644 --- a/sdk/python/docs/source/feast.infra.feature_servers.rst +++ b/sdk/python/docs/source/feast.infra.feature_servers.rst @@ -7,8 +7,6 @@ Subpackages .. toctree:: :maxdepth: 4 - feast.infra.feature_servers.aws_lambda - feast.infra.feature_servers.gcp_cloudrun feast.infra.feature_servers.local_process feast.infra.feature_servers.multicloud diff --git a/sdk/python/docs/source/feast.infra.offline_stores.rst b/sdk/python/docs/source/feast.infra.offline_stores.rst index 052a114cfb..c770e5c13b 100644 --- a/sdk/python/docs/source/feast.infra.offline_stores.rst +++ b/sdk/python/docs/source/feast.infra.offline_stores.rst @@ -28,18 +28,18 @@ feast.infra.offline\_stores.bigquery\_source module :undoc-members: :show-inheritance: -feast.infra.offline\_stores.duckdb module ------------------------------------------ +feast.infra.offline\_stores.dask module +--------------------------------------- -.. automodule:: feast.infra.offline_stores.duckdb +.. automodule:: feast.infra.offline_stores.dask :members: :undoc-members: :show-inheritance: -feast.infra.offline\_stores.file module ---------------------------------------- +feast.infra.offline\_stores.duckdb module +----------------------------------------- -.. automodule:: feast.infra.offline_stores.file +.. automodule:: feast.infra.offline_stores.duckdb :members: :undoc-members: :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.registry.contrib.rst b/sdk/python/docs/source/feast.infra.registry.contrib.rst index 44b89736ad..83417109b8 100644 --- a/sdk/python/docs/source/feast.infra.registry.contrib.rst +++ b/sdk/python/docs/source/feast.infra.registry.contrib.rst @@ -8,7 +8,6 @@ Subpackages :maxdepth: 4 feast.infra.registry.contrib.azure - feast.infra.registry.contrib.postgres Module contents --------------- diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst index a1dfc86492..b0046a2719 100644 --- a/sdk/python/docs/source/feast.infra.rst +++ b/sdk/python/docs/source/feast.infra.rst @@ -19,22 +19,6 @@ Subpackages Submodules ---------- -feast.infra.aws module ----------------------- - -.. automodule:: feast.infra.aws - :members: - :undoc-members: - :show-inheritance: - -feast.infra.gcp module ----------------------- - -.. automodule:: feast.infra.gcp - :members: - :undoc-members: - :show-inheritance: - feast.infra.infra\_object module -------------------------------- @@ -51,14 +35,6 @@ feast.infra.key\_encoding\_utils module :undoc-members: :show-inheritance: -feast.infra.local module ------------------------- - -.. automodule:: feast.infra.local - :members: - :undoc-members: - :show-inheritance: - feast.infra.passthrough\_provider module ---------------------------------------- diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/dask.py similarity index 97% rename from sdk/python/feast/infra/offline_stores/file.py rename to sdk/python/feast/infra/offline_stores/dask.py index af2570ebc0..4a63baf646 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/dask.py @@ -39,20 +39,20 @@ from feast.saved_dataset import SavedDatasetStorage from feast.utils import _get_requested_feature_views_to_features_dict -# FileRetrievalJob will cast string objects to string[pyarrow] from dask version 2023.7.1 +# DaskRetrievalJob will cast string objects to string[pyarrow] from dask version 2023.7.1 # This is not the desired behavior for our use case, so we set the convert-string option to False # See (https://github.com/dask/dask/issues/10881#issuecomment-1923327936) dask.config.set({"dataframe.convert-string": False}) -class FileOfflineStoreConfig(FeastConfigBaseModel): - """Offline store config for local (file-based) store""" +class DaskOfflineStoreConfig(FeastConfigBaseModel): + """Offline store config for dask store""" - type: Literal["file"] = "file" + type: Union[Literal["dask"], Literal["file"]] = "dask" """ Offline store type selector""" -class FileRetrievalJob(RetrievalJob): +class DaskRetrievalJob(RetrievalJob): def __init__( self, evaluation_function: Callable, @@ -122,7 +122,7 @@ def supports_remote_storage_export(self) -> bool: return False -class FileOfflineStore(OfflineStore): +class DaskOfflineStore(OfflineStore): @staticmethod def get_historical_features( config: RepoConfig, @@ -133,7 +133,7 @@ def get_historical_features( project: str, full_feature_names: bool = False, ) -> RetrievalJob: - assert isinstance(config.offline_store, FileOfflineStoreConfig) + assert isinstance(config.offline_store, DaskOfflineStoreConfig) for fv in feature_views: assert isinstance(fv.batch_source, FileSource) @@ -283,7 +283,7 @@ def evaluate_historical_retrieval(): return entity_df_with_features.persist() - job = FileRetrievalJob( + job = DaskRetrievalJob( evaluation_function=evaluate_historical_retrieval, full_feature_names=full_feature_names, on_demand_feature_views=OnDemandFeatureView.get_requested_odfvs( @@ -309,7 +309,7 @@ def pull_latest_from_table_or_query( start_date: datetime, end_date: datetime, ) -> RetrievalJob: - assert isinstance(config.offline_store, FileOfflineStoreConfig) + assert isinstance(config.offline_store, DaskOfflineStoreConfig) assert isinstance(data_source, FileSource) # Create lazy function that is only called from the RetrievalJob object @@ -372,7 +372,7 @@ def evaluate_offline_job(): return source_df[list(columns_to_extract)].persist() # When materializing a single feature view, we don't need full feature names. On demand transforms aren't materialized - return FileRetrievalJob( + return DaskRetrievalJob( evaluation_function=evaluate_offline_job, full_feature_names=False, ) @@ -387,10 +387,10 @@ def pull_all_from_table_or_query( start_date: datetime, end_date: datetime, ) -> RetrievalJob: - assert isinstance(config.offline_store, FileOfflineStoreConfig) + assert isinstance(config.offline_store, DaskOfflineStoreConfig) assert isinstance(data_source, FileSource) - return FileOfflineStore.pull_latest_from_table_or_query( + return DaskOfflineStore.pull_latest_from_table_or_query( config=config, data_source=data_source, join_key_columns=join_key_columns @@ -410,7 +410,7 @@ def write_logged_features( logging_config: LoggingConfig, registry: BaseRegistry, ): - assert isinstance(config.offline_store, FileOfflineStoreConfig) + assert isinstance(config.offline_store, DaskOfflineStoreConfig) destination = logging_config.destination assert isinstance(destination, FileLoggingDestination) @@ -441,7 +441,7 @@ def offline_write_batch( table: pyarrow.Table, progress: Optional[Callable[[int], Any]], ): - assert isinstance(config.offline_store, FileOfflineStoreConfig) + assert isinstance(config.offline_store, DaskOfflineStoreConfig) assert isinstance(feature_view.batch_source, FileSource) pa_schema, column_names = get_pyarrow_schema_from_batch_source( diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 137023ef22..fc2792e323 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -68,7 +68,8 @@ } OFFLINE_STORE_CLASS_FOR_TYPE = { - "file": "feast.infra.offline_stores.file.FileOfflineStore", + "file": "feast.infra.offline_stores.dask.DaskOfflineStore", + "dask": "feast.infra.offline_stores.dask.DaskOfflineStore", "bigquery": "feast.infra.offline_stores.bigquery.BigQueryOfflineStore", "redshift": "feast.infra.offline_stores.redshift.RedshiftOfflineStore", "snowflake.offline": "feast.infra.offline_stores.snowflake.SnowflakeOfflineStore", @@ -205,7 +206,7 @@ def __init__(self, **data: Any): self.registry_config = data["registry"] self._offline_store = None - self.offline_config = data.get("offline_store", "file") + self.offline_config = data.get("offline_store", "dask") self._online_store = None self.online_config = data.get("online_store", "sqlite") @@ -348,7 +349,7 @@ def _validate_offline_store_config(cls, values: Any) -> Any: # Set the default type if "type" not in values["offline_store"]: - values["offline_store"]["type"] = "file" + values["offline_store"]["type"] = "dask" offline_store_type = values["offline_store"]["type"] diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 4a4a7360d8..5174e16046 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -20,8 +20,8 @@ from feast.data_format import DeltaFormat, ParquetFormat from feast.data_source import DataSource from feast.feature_logging import LoggingDestination +from feast.infra.offline_stores.dask import DaskOfflineStoreConfig from feast.infra.offline_stores.duckdb import DuckDBOfflineStoreConfig -from feast.infra.offline_stores.file import FileOfflineStoreConfig from feast.infra.offline_stores.file_source import ( FileLoggingDestination, SavedDatasetFileStorage, @@ -84,7 +84,7 @@ def get_prefixed_table_name(self, suffix: str) -> str: return f"{self.project_name}.{suffix}" def create_offline_store_config(self) -> FeastConfigBaseModel: - return FileOfflineStoreConfig() + return DaskOfflineStoreConfig() def create_logged_features_destination(self) -> LoggingDestination: d = tempfile.mkdtemp(prefix=self.project_name) @@ -334,7 +334,7 @@ def get_prefixed_table_name(self, suffix: str) -> str: return f"{suffix}" def create_offline_store_config(self) -> FeastConfigBaseModel: - return FileOfflineStoreConfig() + return DaskOfflineStoreConfig() def teardown(self): self.minio.stop() diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index 3589c8a3fa..50f048928d 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -23,7 +23,7 @@ from feast.infra.offline_stores.contrib.trino_offline_store.trino import ( TrinoRetrievalJob, ) -from feast.infra.offline_stores.file import FileRetrievalJob +from feast.infra.offline_stores.dask import DaskRetrievalJob from feast.infra.offline_stores.offline_store import RetrievalJob, RetrievalMetadata from feast.infra.offline_stores.redshift import ( RedshiftOfflineStoreConfig, @@ -100,7 +100,7 @@ def metadata(self) -> Optional[RetrievalMetadata]: @pytest.fixture( params=[ MockRetrievalJob, - FileRetrievalJob, + DaskRetrievalJob, RedshiftRetrievalJob, SnowflakeRetrievalJob, AthenaRetrievalJob, @@ -112,8 +112,8 @@ def metadata(self) -> Optional[RetrievalMetadata]: ] ) def retrieval_job(request, environment): - if request.param is FileRetrievalJob: - return FileRetrievalJob(lambda: 1, full_feature_names=False) + if request.param is DaskRetrievalJob: + return DaskRetrievalJob(lambda: 1, full_feature_names=False) elif request.param is RedshiftRetrievalJob: offline_store_config = RedshiftOfflineStoreConfig( cluster_id="feast-int-bucket", diff --git a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py index 6045dbc6ce..6ff7b3c360 100644 --- a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py +++ b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py @@ -5,7 +5,7 @@ import pytest from moto import mock_dynamodb -from feast.infra.offline_stores.file import FileOfflineStoreConfig +from feast.infra.offline_stores.dask import DaskOfflineStoreConfig from feast.infra.online_stores.dynamodb import ( DynamoDBOnlineStore, DynamoDBOnlineStoreConfig, @@ -40,7 +40,7 @@ def repo_config(): provider=PROVIDER, online_store=DynamoDBOnlineStoreConfig(region=REGION), # online_store={"type": "dynamodb", "region": REGION}, - offline_store=FileOfflineStoreConfig(), + offline_store=DaskOfflineStoreConfig(), entity_key_serialization_version=2, ) From 40270e754660d0a8f57cc8a3bbfb1e1e346c3d86 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Mon, 15 Jul 2024 03:47:12 -0400 Subject: [PATCH 25/33] fix: Avoid XSS attack from Jinjin2's Environment(). (#4355) Signed-off-by: Shuchu Han --- .../offline_stores/contrib/postgres_offline_store/postgres.py | 4 +++- sdk/python/feast/infra/offline_stores/offline_utils.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index c4740a960e..c3bbfd97bc 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -365,7 +365,9 @@ def build_point_in_time_query( full_feature_names: bool = False, ) -> str: """Build point-in-time query between each feature view table and the entity dataframe for PostgreSQL""" - template = Environment(loader=BaseLoader()).from_string(source=query_template) + template = Environment(autoescape=True, loader=BaseLoader()).from_string( + source=query_template + ) final_output_feature_names = list(entity_df_columns) final_output_feature_names.extend( diff --git a/sdk/python/feast/infra/offline_stores/offline_utils.py b/sdk/python/feast/infra/offline_stores/offline_utils.py index 2d4fa268e4..6036ba5472 100644 --- a/sdk/python/feast/infra/offline_stores/offline_utils.py +++ b/sdk/python/feast/infra/offline_stores/offline_utils.py @@ -186,7 +186,9 @@ def build_point_in_time_query( full_feature_names: bool = False, ) -> str: """Build point-in-time query between each feature view table and the entity dataframe for Bigquery and Redshift""" - template = Environment(loader=BaseLoader()).from_string(source=query_template) + template = Environment(autoescape=True, loader=BaseLoader()).from_string( + source=query_template + ) final_output_feature_names = list(entity_df_columns) final_output_feature_names.extend( From 38cae164000e116d08bb5b403d573efd03e34b6f Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 16 Jul 2024 09:05:34 +0400 Subject: [PATCH 26/33] chore: Bump google-cloud-datastore lower bound (#4348) bump google-cloud-datastore lower bound Signed-off-by: tokoko --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a543262821..b836200005 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ "googleapis-common-protos>=1.52.0,<2", "google-cloud-bigquery[pandas]>=2,<3.13.0", "google-cloud-bigquery-storage >= 2.0.0,<3", - "google-cloud-datastore>=2.1.0,<3", + "google-cloud-datastore>=2.16.0,<3", "google-cloud-storage>=1.34.0,<3", "google-cloud-bigtable>=2.11.0,<3", "fsspec<=2024.1.0", From cdeab486970ccb8c716499610f927a6e8eb14457 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Tue, 16 Jul 2024 14:47:00 -0400 Subject: [PATCH 27/33] revert: Revert "fix: Avoid XSS attack from Jinjin2's Environment()." (#4357) Revert "fix: Avoid XSS attack from Jinjin2's Environment(). (#4355)" This reverts commit 40270e754660d0a8f57cc8a3bbfb1e1e346c3d86. --- .../offline_stores/contrib/postgres_offline_store/postgres.py | 4 +--- sdk/python/feast/infra/offline_stores/offline_utils.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index c3bbfd97bc..c4740a960e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -365,9 +365,7 @@ def build_point_in_time_query( full_feature_names: bool = False, ) -> str: """Build point-in-time query between each feature view table and the entity dataframe for PostgreSQL""" - template = Environment(autoescape=True, loader=BaseLoader()).from_string( - source=query_template - ) + template = Environment(loader=BaseLoader()).from_string(source=query_template) final_output_feature_names = list(entity_df_columns) final_output_feature_names.extend( diff --git a/sdk/python/feast/infra/offline_stores/offline_utils.py b/sdk/python/feast/infra/offline_stores/offline_utils.py index 6036ba5472..2d4fa268e4 100644 --- a/sdk/python/feast/infra/offline_stores/offline_utils.py +++ b/sdk/python/feast/infra/offline_stores/offline_utils.py @@ -186,9 +186,7 @@ def build_point_in_time_query( full_feature_names: bool = False, ) -> str: """Build point-in-time query between each feature view table and the entity dataframe for Bigquery and Redshift""" - template = Environment(autoescape=True, loader=BaseLoader()).from_string( - source=query_template - ) + template = Environment(loader=BaseLoader()).from_string(source=query_template) final_output_feature_names = list(entity_df_columns) final_output_feature_names.extend( From ce4f09b9d21f0b9315f1b8b79772901d2081813d Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 16 Jul 2024 23:05:40 +0400 Subject: [PATCH 28/33] chore: Remove distutils (#4356) * remove distutils Signed-off-by: tokoko * fix formatting Signed-off-by: tokoko --------- Signed-off-by: tokoko --- sdk/python/feast/repo_operations.py | 4 ++-- setup.py | 19 ++++++------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 05a7d05e23..a3100ca9d7 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -384,8 +384,8 @@ def cli_check_repo(repo_path: Path, fs_yaml_file: Path): def init_repo(repo_name: str, template: str): import os - from distutils.dir_util import copy_tree from pathlib import Path + from shutil import copytree from colorama import Fore, Style @@ -412,7 +412,7 @@ def init_repo(repo_name: str, template: str): template_path = str(Path(Path(__file__).parent / "templates" / template).absolute()) if not os.path.exists(template_path): raise IOError(f"Could not find template {template}") - copy_tree(template_path, str(repo_path)) + copytree(template_path, str(repo_path), dirs_exist_ok=True) # Seed the repository bootstrap_path = repo_path / "bootstrap.py" diff --git a/setup.py b/setup.py index b836200005..4ac492b3bf 100644 --- a/setup.py +++ b/setup.py @@ -18,21 +18,14 @@ import shutil import subprocess import sys -from distutils.cmd import Command -from pathlib import Path - -from setuptools import find_packages -try: - from setuptools import setup - from setuptools.command.build_ext import build_ext as _build_ext - from setuptools.command.build_py import build_py - from setuptools.command.develop import develop - from setuptools.command.install import install +from pathlib import Path -except ImportError: - from distutils.command.build_py import build_py - from distutils.core import setup +from setuptools import find_packages, setup, Command +from setuptools.command.build_ext import build_ext as _build_ext +from setuptools.command.build_py import build_py +from setuptools.command.develop import develop +from setuptools.command.install import install NAME = "feast" DESCRIPTION = "Python SDK for Feast" From a8bc696010fa94fa0be44fba2570bee0eab83ba2 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Tue, 16 Jul 2024 15:22:30 -0400 Subject: [PATCH 29/33] fix: Retire the datetime.utcnow(). (#4352) * fix: Retire the datetime.utcnow(). Signed-off-by: Shuchu Han * fix: Remove unnecessary unit test. Signed-off-by: Shuchu Han --------- Signed-off-by: Shuchu Han --- sdk/python/feast/driver_test_data.py | 43 ++++++++++++++++------ sdk/python/feast/type_map.py | 3 +- sdk/python/feast/utils.py | 4 +- sdk/python/tests/unit/test_datetime.py | 6 --- sdk/python/tests/utils/feature_records.py | 4 +- sdk/python/tests/utils/test_log_creator.py | 2 +- 6 files changed, 38 insertions(+), 24 deletions(-) delete mode 100644 sdk/python/tests/unit/test_datetime.py diff --git a/sdk/python/feast/driver_test_data.py b/sdk/python/feast/driver_test_data.py index 7959046e6e..defeb404a3 100644 --- a/sdk/python/feast/driver_test_data.py +++ b/sdk/python/feast/driver_test_data.py @@ -61,11 +61,11 @@ def create_orders_df( df["order_is_success"] = np.random.randint(0, 2, size=order_count).astype(np.int32) df[DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL] = [ _convert_event_timestamp( - pd.Timestamp(dt, unit="ms", tz="UTC").round("ms"), + pd.Timestamp(dt, unit="ms").round("ms"), EventTimestampType(idx % 4), ) for idx, dt in enumerate( - pd.date_range(start=start_date, end=end_date, periods=order_count) + pd.date_range(start=start_date, end=end_date, periods=order_count, tz="UTC") ) ] df.sort_values( @@ -101,9 +101,13 @@ def create_driver_hourly_stats_df(drivers, start_date, end_date) -> pd.DataFrame df_hourly = pd.DataFrame( { "event_timestamp": [ - pd.Timestamp(dt, unit="ms", tz="UTC").round("ms") + pd.Timestamp(dt, unit="ms").round("ms") for dt in pd.date_range( - start=start_date, end=end_date, freq="1h", inclusive="left" + start=start_date, + end=end_date, + freq="1h", + inclusive="left", + tz="UTC", ) ] # include a fixed timestamp for get_historical_features in the quickstart @@ -162,9 +166,13 @@ def create_customer_daily_profile_df(customers, start_date, end_date) -> pd.Data df_daily = pd.DataFrame( { "event_timestamp": [ - pd.Timestamp(dt, unit="ms", tz="UTC").round("ms") + pd.Timestamp(dt, unit="ms").round("ms") for dt in pd.date_range( - start=start_date, end=end_date, freq="1D", inclusive="left" + start=start_date, + end=end_date, + freq="1D", + inclusive="left", + tz="UTC", ) ] } @@ -207,9 +215,13 @@ def create_location_stats_df(locations, start_date, end_date) -> pd.DataFrame: df_hourly = pd.DataFrame( { "event_timestamp": [ - pd.Timestamp(dt, unit="ms", tz="UTC").round("ms") + pd.Timestamp(dt, unit="ms").round("ms") for dt in pd.date_range( - start=start_date, end=end_date, freq="1h", inclusive="left" + start=start_date, + end=end_date, + freq="1h", + inclusive="left", + tz="UTC", ) ] } @@ -254,9 +266,16 @@ def create_global_daily_stats_df(start_date, end_date) -> pd.DataFrame: df_daily = pd.DataFrame( { "event_timestamp": [ - pd.Timestamp(dt, unit="ms", tz="UTC").round("ms") + pd.Timestamp( + dt, + unit="ms", + ).round("ms") for dt in pd.date_range( - start=start_date, end=end_date, freq="1D", inclusive="left" + start=start_date, + end=end_date, + freq="1D", + inclusive="left", + tz="UTC", ) ] } @@ -286,11 +305,11 @@ def create_field_mapping_df(start_date, end_date) -> pd.DataFrame: df["column_name"] = np.random.randint(1, 100, size=size).astype(np.int32) df[DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL] = [ _convert_event_timestamp( - pd.Timestamp(dt, unit="ms", tz="UTC").round("ms"), + pd.Timestamp(dt, unit="ms").round("ms"), EventTimestampType(idx % 4), ) for idx, dt in enumerate( - pd.date_range(start=start_date, end=end_date, periods=size) + pd.date_range(start=start_date, end=end_date, periods=size, tz="UTC") ) ] df["created"] = pd.to_datetime(pd.Timestamp.now(tz=None).round("ms")) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 6ba61fc8c5..703c1dc7c5 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -162,7 +162,8 @@ def python_type_to_feast_value_type( "timestamp": ValueType.UNIX_TIMESTAMP, "datetime": ValueType.UNIX_TIMESTAMP, "datetime64[ns]": ValueType.UNIX_TIMESTAMP, - "datetime64[ns, tz]": ValueType.UNIX_TIMESTAMP, + "datetime64[ns, tz]": ValueType.UNIX_TIMESTAMP, # special dtype of pandas + "datetime64[ns, utc]": ValueType.UNIX_TIMESTAMP, "category": ValueType.STRING, } diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 1a1d757fc1..0467393aa2 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -5,7 +5,7 @@ import typing import warnings from collections import Counter, defaultdict -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import ( Any, @@ -1055,4 +1055,4 @@ def tags_str_to_dict(tags: str = "") -> dict[str, str]: def _utc_now() -> datetime: - return datetime.utcnow() + return datetime.now(tz=timezone.utc) diff --git a/sdk/python/tests/unit/test_datetime.py b/sdk/python/tests/unit/test_datetime.py deleted file mode 100644 index aaab507ed0..0000000000 --- a/sdk/python/tests/unit/test_datetime.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- - - -""" -Test the retirement of datetime.utcnow() function. -""" diff --git a/sdk/python/tests/utils/feature_records.py b/sdk/python/tests/utils/feature_records.py index 2c26f3c000..bd3567c9ee 100644 --- a/sdk/python/tests/utils/feature_records.py +++ b/sdk/python/tests/utils/feature_records.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional import numpy as np @@ -520,7 +520,7 @@ def get_last_feature_row(df: pd.DataFrame, driver_id, max_date: datetime): """Manually extract last feature value from a dataframe for a given driver_id with up to `max_date` date""" filtered = df[ (df["driver_id"] == driver_id) - & (df["event_timestamp"] < max_date.replace(tzinfo=utc)) + & (df["event_timestamp"] < max_date.replace(tzinfo=timezone.utc)) ] max_ts = filtered.loc[filtered["event_timestamp"].idxmax()]["event_timestamp"] filtered_by_ts = filtered[filtered["event_timestamp"] == max_ts] diff --git a/sdk/python/tests/utils/test_log_creator.py b/sdk/python/tests/utils/test_log_creator.py index f072f4c886..987c8d77ef 100644 --- a/sdk/python/tests/utils/test_log_creator.py +++ b/sdk/python/tests/utils/test_log_creator.py @@ -117,7 +117,7 @@ def prepare_logs( f"{destination_field}__status" ].mask( logs_df[f"{destination_field}__timestamp"] - < (_utc_now() - view.ttl), + < (_utc_now() - view.ttl).replace(tzinfo=None), FieldStatus.OUTSIDE_MAX_AGE, ) From 5a1636431d44a7e109c64688fe3176741feccc1b Mon Sep 17 00:00:00 2001 From: camenares <32527085+camenares@users.noreply.github.com> Date: Tue, 16 Jul 2024 17:13:20 -0400 Subject: [PATCH 30/33] chore: Change arrow scalar ids usage (#4347) * Update google-cloud-storage Signed-off-by: Christopher Camenares * test tighter library restriction Signed-off-by: Christopher Camenares * fix lint Signed-off-by: Christopher Camenares * bump <4 again Signed-off-by: Christopher Camenares --------- Signed-off-by: Christopher Camenares --- .../feast/infra/offline_stores/bigquery.py | 15 +- .../requirements/py3.10-ci-requirements.txt | 152 +++++++++++++++--- .../requirements/py3.10-requirements.txt | 48 ++++-- .../requirements/py3.11-ci-requirements.txt | 152 +++++++++++++++--- .../requirements/py3.11-requirements.txt | 48 ++++-- .../requirements/py3.9-ci-requirements.txt | 152 +++++++++++++++--- .../requirements/py3.9-requirements.txt | 48 ++++-- setup.py | 2 +- 8 files changed, 532 insertions(+), 85 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 3e4a0f1b99..ef12eba442 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -59,7 +59,6 @@ from google.auth.exceptions import DefaultCredentialsError from google.cloud import bigquery from google.cloud.bigquery import Client, SchemaField, Table - from google.cloud.bigquery._pandas_helpers import ARROW_SCALAR_IDS_TO_BQ from google.cloud.storage import Client as StorageClient except ImportError as e: @@ -67,6 +66,16 @@ raise FeastExtrasDependencyImportError("gcp", str(e)) +try: + from google.cloud.bigquery._pyarrow_helpers import _ARROW_SCALAR_IDS_TO_BQ +except ImportError: + try: + from google.cloud.bigquery._pandas_helpers import ( # type: ignore + ARROW_SCALAR_IDS_TO_BQ as _ARROW_SCALAR_IDS_TO_BQ, + ) + except ImportError as e: + raise FeastExtrasDependencyImportError("gcp", str(e)) + def get_http_client_info(): return http_client_info.ClientInfo(user_agent=get_user_agent()) @@ -794,10 +803,10 @@ def arrow_schema_to_bq_schema(arrow_schema: pyarrow.Schema) -> List[SchemaField] for field in arrow_schema: if pyarrow.types.is_list(field.type): detected_mode = "REPEATED" - detected_type = ARROW_SCALAR_IDS_TO_BQ[field.type.value_type.id] + detected_type = _ARROW_SCALAR_IDS_TO_BQ[field.type.value_type.id] else: detected_mode = "NULLABLE" - detected_type = ARROW_SCALAR_IDS_TO_BQ[field.type.id] + detected_type = _ARROW_SCALAR_IDS_TO_BQ[field.type.id] bq_schema.append( SchemaField(name=field.name, field_type=detected_type, mode=detected_mode) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 33709a1ef0..a7f300a0ed 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.13.1 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -19,6 +20,8 @@ anyio==4.4.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -48,7 +52,9 @@ azure-core==1.30.2 # azure-identity # azure-storage-blob azure-identity==1.17.1 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -60,7 +66,9 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.131 - # via moto + # via + # feast (setup.py) + # moto botocore==1.34.131 # via # aiobotocore @@ -69,6 +77,7 @@ botocore==1.34.131 # s3transfer build==1.2.1 # via + # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -76,6 +85,7 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.7.4 # via # elastic-transport @@ -98,6 +108,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -107,7 +118,9 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel @@ -116,6 +129,7 @@ coverage[toml]==7.5.4 # via pytest-cov cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -127,7 +141,9 @@ cryptography==42.0.8 # types-pyopenssl # types-redis dask[dataframe]==2024.6.2 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.6 # via dask db-dtypes==1.2.0 @@ -139,7 +155,9 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.18.1 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -153,6 +171,7 @@ duckdb==0.10.3 elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.14.0 + # via feast (setup.py) email-validator==2.2.0 # via fastapi entrypoints==0.4 @@ -167,6 +186,7 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 + # via feast (setup.py) fastapi-cli==0.0.4 # via fastapi fastjsonschema==2.20.0 @@ -176,6 +196,7 @@ filelock==3.15.4 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -183,13 +204,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.1 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -213,9 +237,12 @@ google-auth==2.30.0 # kubernetes google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 +google-cloud-bigquery[pandas]==3.13.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) google-cloud-bigtable==2.24.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -224,10 +251,13 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.17.0 - # via firebase-admin + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -238,16 +268,17 @@ google-resumable-media==2.7.1 # google-cloud-storage googleapis-common-protos[grpc]==1.63.2 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.16 -greenlet==3.0.3 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -258,19 +289,27 @@ grpcio==1.64.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -281,11 +320,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==9.1.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -320,6 +363,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -343,6 +387,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -388,6 +433,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -408,13 +454,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.29.0 # via # azure-identity @@ -428,10 +478,13 @@ multidict==6.0.5 # aiohttp # yarl mypy==1.10.1 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -454,6 +507,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -488,6 +542,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -513,6 +568,7 @@ pexpect==4.9.0 pip==24.1.1 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -525,6 +581,7 @@ ply==3.11 portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.47 @@ -539,6 +596,7 @@ proto-plus==1.24.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -556,8 +614,11 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel + # via + # feast (setup.py) + # ipykernel psycopg[binary, pool]==3.1.19 + # via feast (setup.py) psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.2 @@ -569,12 +630,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -592,16 +655,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.4 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -612,8 +678,11 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -625,8 +694,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -636,13 +707,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -671,6 +750,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -684,15 +764,19 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -727,6 +811,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -736,6 +821,7 @@ rsa==4.9 ruamel-yaml==0.17.17 # via great-expectations ruff==0.4.10 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.0 @@ -752,6 +838,7 @@ setuptools==70.1.1 shellingham==1.5.4 # via typer singlestoredb==1.4.0 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -772,11 +859,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.11.0 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -790,9 +879,11 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.31 + # via feast (setup.py) sqlglot==25.1.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -802,17 +893,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.4.2 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -840,7 +935,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -857,25 +954,39 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==70.1.0.20240627 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -914,6 +1025,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -925,11 +1037,15 @@ urllib3==1.26.19 # rockset # testcontainers uvicorn[standard]==0.30.1 - # via fastapi + # via + # feast (setup.py) + # fastapi uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 0cca106863..39a278818f 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -65,8 +71,11 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -78,13 +87,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -96,20 +108,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -120,6 +141,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -127,6 +149,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -142,11 +165,15 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -154,7 +181,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -178,6 +207,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 09e9e8eeea..4c1be0a5b4 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.13.1 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -19,6 +20,8 @@ anyio==4.4.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -44,7 +48,9 @@ azure-core==1.30.2 # azure-identity # azure-storage-blob azure-identity==1.17.1 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -56,7 +62,9 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.131 - # via moto + # via + # feast (setup.py) + # moto botocore==1.34.131 # via # aiobotocore @@ -65,6 +73,7 @@ botocore==1.34.131 # s3transfer build==1.2.1 # via + # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -72,6 +81,7 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.7.4 # via # elastic-transport @@ -94,6 +104,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -103,7 +114,9 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel @@ -112,6 +125,7 @@ coverage[toml]==7.5.4 # via pytest-cov cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -123,7 +137,9 @@ cryptography==42.0.8 # types-pyopenssl # types-redis dask[dataframe]==2024.6.2 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.6 # via dask db-dtypes==1.2.0 @@ -135,7 +151,9 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.18.1 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -149,6 +167,7 @@ duckdb==0.10.3 elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.14.0 + # via feast (setup.py) email-validator==2.2.0 # via fastapi entrypoints==0.4 @@ -158,6 +177,7 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 + # via feast (setup.py) fastapi-cli==0.0.4 # via fastapi fastjsonschema==2.20.0 @@ -167,6 +187,7 @@ filelock==3.15.4 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -174,13 +195,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.1 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -204,9 +228,12 @@ google-auth==2.30.0 # kubernetes google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 +google-cloud-bigquery[pandas]==3.13.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) google-cloud-bigtable==2.24.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -215,10 +242,13 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.17.0 - # via firebase-admin + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -229,16 +259,17 @@ google-resumable-media==2.7.1 # google-cloud-storage googleapis-common-protos[grpc]==1.63.2 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.16 -greenlet==3.0.3 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -249,19 +280,27 @@ grpcio==1.64.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -272,11 +311,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==9.1.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -311,6 +354,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -334,6 +378,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -379,6 +424,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -399,13 +445,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.29.0 # via # azure-identity @@ -419,10 +469,13 @@ multidict==6.0.5 # aiohttp # yarl mypy==1.10.1 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -445,6 +498,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -479,6 +533,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -504,6 +559,7 @@ pexpect==4.9.0 pip==24.1.1 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -516,6 +572,7 @@ ply==3.11 portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.47 @@ -530,6 +587,7 @@ proto-plus==1.24.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -547,8 +605,11 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel + # via + # feast (setup.py) + # ipykernel psycopg[binary, pool]==3.1.19 + # via feast (setup.py) psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.2 @@ -560,12 +621,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -583,16 +646,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.4 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -603,8 +669,11 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -616,8 +685,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -627,13 +698,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -662,6 +741,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -675,15 +755,19 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -718,6 +802,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -727,6 +812,7 @@ rsa==4.9 ruamel-yaml==0.17.17 # via great-expectations ruff==0.4.10 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.0 @@ -743,6 +829,7 @@ setuptools==70.1.1 shellingham==1.5.4 # via typer singlestoredb==1.4.0 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -763,11 +850,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.11.0 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -781,9 +870,11 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.31 + # via feast (setup.py) sqlglot==25.1.0 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -793,17 +884,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.4.2 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 @@ -821,7 +916,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -838,25 +935,39 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==70.1.0.20240627 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -892,6 +1003,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -903,11 +1015,15 @@ urllib3==1.26.19 # rockset # testcontainers uvicorn[standard]==0.30.1 - # via fastapi + # via + # feast (setup.py) + # fastapi uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 687e4bfe52..44e658113a 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -20,30 +20,36 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -63,8 +69,11 @@ idna==3.7 importlib-metadata==7.1.0 # via dask jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -76,13 +85,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -94,20 +106,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -118,6 +139,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -125,6 +147,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -140,17 +163,23 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) toolz==0.12.1 # via # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -172,6 +201,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 6f5d0220bc..25cdea7a68 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.13.1 + # via feast (setup.py) aiohttp==3.9.5 # via aiobotocore aioitertools==0.11.0 @@ -19,6 +20,8 @@ anyio==4.4.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -28,6 +31,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -48,7 +52,9 @@ azure-core==1.30.2 # azure-identity # azure-storage-blob azure-identity==1.17.1 + # via feast (setup.py) azure-storage-blob==12.20.0 + # via feast (setup.py) babel==2.15.0 # via # jupyterlab-server @@ -60,7 +66,9 @@ bidict==0.23.1 bleach==6.1.0 # via nbconvert boto3==1.34.131 - # via moto + # via + # feast (setup.py) + # moto botocore==1.34.131 # via # aiobotocore @@ -69,6 +77,7 @@ botocore==1.34.131 # s3transfer build==1.2.1 # via + # feast (setup.py) # pip-tools # singlestoredb cachecontrol==0.14.0 @@ -76,6 +85,7 @@ cachecontrol==0.14.0 cachetools==5.3.3 # via google-auth cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.7.4 # via # elastic-transport @@ -98,6 +108,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -107,7 +118,9 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel @@ -116,6 +129,7 @@ coverage[toml]==7.5.4 # via pytest-cov cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -127,7 +141,9 @@ cryptography==42.0.8 # types-pyopenssl # types-redis dask[dataframe]==2024.6.2 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.6 # via dask db-dtypes==1.2.0 @@ -139,7 +155,9 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.18.1 + # via feast (setup.py) dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -153,6 +171,7 @@ duckdb==0.10.3 elastic-transport==8.13.1 # via elasticsearch elasticsearch==8.14.0 + # via feast (setup.py) email-validator==2.2.0 # via fastapi entrypoints==0.4 @@ -167,6 +186,7 @@ execnet==2.1.1 executing==2.0.1 # via stack-data fastapi==0.111.0 + # via feast (setup.py) fastapi-cli==0.0.4 # via fastapi fastjsonschema==2.20.0 @@ -176,6 +196,7 @@ filelock==3.15.4 # snowflake-connector-python # virtualenv firebase-admin==5.4.0 + # via feast (setup.py) fqdn==1.5.1 # via jsonschema frozenlist==1.4.1 @@ -183,13 +204,16 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.19.1 # via + # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery @@ -213,9 +237,12 @@ google-auth==2.30.0 # kubernetes google-auth-httplib2==0.2.0 # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 +google-cloud-bigquery[pandas]==3.13.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) google-cloud-bigtable==2.24.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -224,10 +251,13 @@ google-cloud-core==2.4.1 # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.19.0 + # via feast (setup.py) google-cloud-firestore==2.16.0 # via firebase-admin google-cloud-storage==2.17.0 - # via firebase-admin + # via + # feast (setup.py) + # firebase-admin google-crc32c==1.5.0 # via # google-cloud-storage @@ -238,16 +268,17 @@ google-resumable-media==2.7.1 # google-cloud-storage googleapis-common-protos[grpc]==1.63.2 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.16 -greenlet==3.0.3 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.64.1 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos @@ -258,19 +289,27 @@ grpcio==1.64.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.2 + # via feast (setup.py) grpcio-reflection==1.62.2 + # via feast (setup.py) grpcio-status==1.62.2 # via google-api-core grpcio-testing==1.62.2 + # via feast (setup.py) grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.4.0 + # via feast (setup.py) hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx httplib2==0.22.0 @@ -281,11 +320,15 @@ httptools==0.6.1 # via uvicorn httpx==0.27.0 # via + # feast (setup.py) # fastapi # jupyterlab ibis-framework[duckdb]==9.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.0 + # via feast (setup.py) identify==2.5.36 # via pre-commit idna==3.7 @@ -329,6 +372,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # fastapi # great-expectations @@ -352,6 +396,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.22.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -397,6 +442,7 @@ jupyterlab-server==2.27.2 jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.2 @@ -417,13 +463,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==4.1.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.29.0 # via # azure-identity @@ -437,10 +487,13 @@ multidict==6.0.5 # aiohttp # yarl mypy==1.10.1 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -463,6 +516,7 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes @@ -497,6 +551,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -522,6 +577,7 @@ pexpect==4.9.0 pip==24.1.1 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -534,6 +590,7 @@ ply==3.11 portalocker==2.10.0 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server prompt-toolkit==3.0.47 @@ -548,6 +605,7 @@ proto-plus==1.24.0 # google-cloud-firestore protobuf==4.25.3 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -565,8 +623,11 @@ protobuf==4.25.3 # proto-plus # substrait psutil==5.9.0 - # via ipykernel + # via + # feast (setup.py) + # ipykernel psycopg[binary, pool]==3.1.18 + # via feast (setup.py) psycopg-binary==3.1.18 # via psycopg psycopg-pool==3.2.2 @@ -578,12 +639,14 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -601,16 +664,19 @@ pyasn1==0.6.0 pyasn1-modules==0.4.0 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.7.4 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.18.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich @@ -621,8 +687,11 @@ pyjwt[crypto]==2.8.0 # singlestoredb # snowflake-connector-python pymssql==2.3.0 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.1.0 # via snowflake-connector-python pyparsing==3.1.2 @@ -634,8 +703,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.1 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -645,13 +716,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -680,6 +759,7 @@ pytz==2024.1 # trino pyyaml==6.0.1 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -693,15 +773,19 @@ pyzmq==26.0.3 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.5.15 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # cachecontrol # docker @@ -736,6 +820,7 @@ rich==13.7.1 # ibis-framework # typer rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema @@ -747,6 +832,7 @@ ruamel-yaml==0.17.17 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.4.10 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.13.1 @@ -763,6 +849,7 @@ setuptools==70.1.1 shellingham==1.5.4 # via typer singlestoredb==1.4.0 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -783,11 +870,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.11.0 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 @@ -801,9 +890,11 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==2.0.31 + # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework sqlite-vec==0.0.1a10 + # via feast (setup.py) sqlparams==6.0.1 # via singlestoredb stack-data==0.6.3 @@ -813,17 +904,21 @@ starlette==0.37.2 substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.4.2 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.1 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -851,7 +946,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.4 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -868,25 +965,39 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.328.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.1.0.20240417 + # via feast (setup.py) types-pyyaml==6.0.12.20240311 + # via feast (setup.py) types-redis==4.6.0.20240425 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==70.1.0.20240627 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -927,6 +1038,7 @@ uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.19 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -939,11 +1051,15 @@ urllib3==1.26.19 # snowflake-connector-python # testcontainers uvicorn[standard]==0.30.1 - # via fastapi + # via + # feast (setup.py) + # fastapi uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.22.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 096f54ab1f..ea553bcae2 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -67,8 +73,11 @@ importlib-metadata==7.1.0 # dask # typeguard jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -80,13 +89,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -98,20 +110,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -122,6 +143,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -129,6 +151,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -144,11 +167,15 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -156,7 +183,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -181,6 +210,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/setup.py b/setup.py index 4ac492b3bf..2043bf1b3f 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ GCP_REQUIRED = [ "google-api-core>=1.23.0,<3", "googleapis-common-protos>=1.52.0,<2", - "google-cloud-bigquery[pandas]>=2,<3.13.0", + "google-cloud-bigquery[pandas]>=2,<4", "google-cloud-bigquery-storage >= 2.0.0,<3", "google-cloud-datastore>=2.16.0,<3", "google-cloud-storage>=1.34.0,<3", From 7a45c5c38a2f8d9b03e79586156ac391a986542b Mon Sep 17 00:00:00 2001 From: Bhargav Dodla Date: Thu, 18 Jul 2024 12:48:29 -0700 Subject: [PATCH 31/33] fix: Fixed formatting issues --- sdk/python/feast/repo_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 09d97e2974..7f97f478ba 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -380,10 +380,10 @@ def _validate_offline_store_config(cls, values: Any) -> Any: # Set the default type if "type" not in values["offline_store"]: - if values["provider"] == "expedia": - values["offline_store"]["type"] = "spark" - else: - values["offline_store"]["type"] = "dask" + if values["provider"] == "expedia": + values["offline_store"]["type"] = "spark" + else: + values["offline_store"]["type"] = "dask" offline_store_type = values["offline_store"]["type"] From 550fe9d86a70a0af6ce6224b6da503777cbb6437 Mon Sep 17 00:00:00 2001 From: Bhargav Dodla Date: Thu, 18 Jul 2024 13:31:18 -0700 Subject: [PATCH 32/33] fix: Fixed test failures --- sdk/python/feast/repo_config.py | 13 ++++++++----- .../tests/expediagroup/test_milvus_online_store.py | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 7f97f478ba..001cb74156 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -218,7 +218,8 @@ def __init__(self, **data: Any): self.registry_config = data["registry"] self._offline_store = None - if data["provider"] == "expedia": + provider = data.get("provider", "local") + if provider == "expedia": spark_offline_config = { "type": "spark", "spark_conf": { @@ -232,7 +233,7 @@ def __init__(self, **data: Any): self.offline_config = data.get("offline_store", "dask") self._online_store = None - if data["provider"] == "expedia": + if provider == "expedia": self.online_config = "redis" else: self.online_config = data.get("online_store", "sqlite") @@ -242,7 +243,7 @@ def __init__(self, **data: Any): self.batch_engine_config = data["batch_engine"] elif "batch_engine_config" in data: self.batch_engine_config = data["batch_engine_config"] - elif data["provider"] == "expedia": + elif provider == "expedia": self.batch_engine_config = "spark.engine" else: # Defaults to using local in-process materialization engine. @@ -351,8 +352,9 @@ def _validate_online_store_config(cls, values: Any) -> Any: # Set the default type # This is only direct reference to a provider or online store that we should have # for backwards compatibility. + provider = values.get("provider", "local") if "type" not in values["online_store"]: - if values["provider"] == "expedia": + if provider == "expedia": values["online_store"]["type"] = "redis" else: values["online_store"]["type"] = "sqlite" @@ -378,9 +380,10 @@ def _validate_offline_store_config(cls, values: Any) -> Any: if not isinstance(values["offline_store"], Dict): return values + provider = values.get("provider", "local") # Set the default type if "type" not in values["offline_store"]: - if values["provider"] == "expedia": + if provider == "expedia": values["offline_store"]["type"] = "spark" else: values["offline_store"]["type"] = "dask" diff --git a/sdk/python/tests/expediagroup/test_milvus_online_store.py b/sdk/python/tests/expediagroup/test_milvus_online_store.py index 0410347d3e..d692747ca4 100644 --- a/sdk/python/tests/expediagroup/test_milvus_online_store.py +++ b/sdk/python/tests/expediagroup/test_milvus_online_store.py @@ -12,6 +12,7 @@ connections, utility, ) +from tests.expediagroup.milvus_online_store_creator import MilvusOnlineStoreCreator from feast import FeatureView from feast.entity import Entity @@ -21,14 +22,13 @@ MilvusOnlineStoreConfig, ) from feast.field import Field -from feast.infra.offline_stores.file import FileOfflineStoreConfig +from feast.infra.offline_stores.dask import DaskOfflineStoreConfig from feast.infra.offline_stores.file_source import FileSource from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import FloatList from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RepoConfig from feast.types import Array, Bytes, Float32, Int64, String -from tests.expediagroup.milvus_online_store_creator import MilvusOnlineStoreCreator logging.basicConfig(level=logging.INFO) @@ -59,7 +59,7 @@ def repo_config(embedded_milvus): username=embedded_milvus["username"], password=embedded_milvus["password"], ), - offline_store=FileOfflineStoreConfig(), + offline_store=DaskOfflineStoreConfig(), entity_key_serialization_version=2, ) From f1b1ced895b27e986cb316bf48f3663402ca6e58 Mon Sep 17 00:00:00 2001 From: Bhargav Dodla Date: Thu, 18 Jul 2024 13:40:19 -0700 Subject: [PATCH 33/33] chore: Fixed Lint issues --- sdk/python/tests/expediagroup/test_milvus_online_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/expediagroup/test_milvus_online_store.py b/sdk/python/tests/expediagroup/test_milvus_online_store.py index d692747ca4..752a0b7e6e 100644 --- a/sdk/python/tests/expediagroup/test_milvus_online_store.py +++ b/sdk/python/tests/expediagroup/test_milvus_online_store.py @@ -12,7 +12,6 @@ connections, utility, ) -from tests.expediagroup.milvus_online_store_creator import MilvusOnlineStoreCreator from feast import FeatureView from feast.entity import Entity @@ -29,6 +28,7 @@ from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RepoConfig from feast.types import Array, Bytes, Float32, Int64, String +from tests.expediagroup.milvus_online_store_creator import MilvusOnlineStoreCreator logging.basicConfig(level=logging.INFO)