From c87fc8f18a24f14d21c1d988508e2dc5a30ee6b0 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin Date: Tue, 8 Oct 2024 11:11:10 +0100 Subject: [PATCH] chore: Prepare 3.10.0 release --- .bumpversion.toml | 2 +- README.md | 80 +++++++++---------- VERSION | 2 +- awswrangler/__metadata__.py | 2 +- awswrangler/athena/_read.py | 16 ++-- awswrangler/catalog/_create.py | 4 +- awswrangler/s3/_read_orc.py | 4 +- awswrangler/s3/_read_parquet.py | 4 +- awswrangler/s3/_read_text.py | 6 +- awswrangler/s3/_write_orc.py | 4 +- awswrangler/s3/_write_parquet.py | 6 +- awswrangler/s3/_write_text.py | 12 +-- docs/source/install.rst | 2 +- pyproject.toml | 2 +- test_infra/pyproject.toml | 2 +- tests/unit/test_metadata.py | 2 +- tutorials/001 - Introduction.ipynb | 20 ++--- ...ySQL, PostgreSQL, SQL Server, Oracle.ipynb | 30 +++---- tutorials/014 - Schema Evolution.ipynb | 6 +- tutorials/021 - Global Configurations.ipynb | 2 +- ...22 - Writing Partitions Concurrently.ipynb | 2 +- .../023 - Flexible Partitions Filter.ipynb | 2 +- tutorials/030 - Data Api.ipynb | 8 +- 23 files changed, 110 insertions(+), 110 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 37afb09a8..fd644f459 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "3.9.2b1" +current_version = "3.10.0" commit = false tag = false tag_name = "{new_version}" diff --git a/README.md b/README.md index 5f63717ae..3c30eacf6 100644 --- a/README.md +++ b/README.md @@ -94,27 +94,27 @@ FROM "sampleDB"."sampleTable" ORDER BY time DESC LIMIT 3 ## At scale AWS SDK for pandas can also run your workflows at scale by leveraging [Modin](https://modin.readthedocs.io/en/stable/) and [Ray](https://www.ray.io/). Both projects aim to speed up data workloads by distributing processing over a cluster of workers. -Read our [docs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/scale.html) or head to our latest [tutorials](https://github.com/aws/aws-sdk-pandas/tree/main/tutorials) to learn more. +Read our [docs](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/scale.html) or head to our latest [tutorials](https://github.com/aws/aws-sdk-pandas/tree/main/tutorials) to learn more. > ⚠️ **Ray is currently not available for Python 3.12. While AWS SDK for pandas supports Python 3.12, it cannot be used at scale.** ## [Read The Docs](https://aws-sdk-pandas.readthedocs.io/) -- [**What is AWS SDK for pandas?**](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/about.html) -- [**Install**](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html) - - [PyPi (pip)](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#pypi-pip) - - [Conda](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#conda) - - [AWS Lambda Layer](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#aws-lambda-layer) - - [AWS Glue Python Shell Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#aws-glue-python-shell-jobs) - - [AWS Glue PySpark Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#aws-glue-pyspark-jobs) - - [Amazon SageMaker Notebook](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#amazon-sagemaker-notebook) - - [Amazon SageMaker Notebook Lifecycle](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#amazon-sagemaker-notebook-lifecycle) - - [EMR](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#emr) - - [From source](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#from-source) -- [**At scale**](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/scale.html) - - [Getting Started](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/scale.html#getting-started) - - [Supported APIs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/scale.html#supported-apis) - - [Resources](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/scale.html#resources) +- [**What is AWS SDK for pandas?**](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/about.html) +- [**Install**](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html) + - [PyPi (pip)](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#pypi-pip) + - [Conda](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#conda) + - [AWS Lambda Layer](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#aws-lambda-layer) + - [AWS Glue Python Shell Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#aws-glue-python-shell-jobs) + - [AWS Glue PySpark Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#aws-glue-pyspark-jobs) + - [Amazon SageMaker Notebook](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#amazon-sagemaker-notebook) + - [Amazon SageMaker Notebook Lifecycle](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#amazon-sagemaker-notebook-lifecycle) + - [EMR](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#emr) + - [From source](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#from-source) +- [**At scale**](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/scale.html) + - [Getting Started](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/scale.html#getting-started) + - [Supported APIs](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/scale.html#supported-apis) + - [Resources](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/scale.html#resources) - [**Tutorials**](https://github.com/aws/aws-sdk-pandas/tree/main/tutorials) - [001 - Introduction](https://github.com/aws/aws-sdk-pandas/blob/main/tutorials/001%20-%20Introduction.ipynb) - [002 - Sessions](https://github.com/aws/aws-sdk-pandas/blob/main/tutorials/002%20-%20Sessions.ipynb) @@ -155,30 +155,30 @@ Read our [docs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/scale.html) or - [039 - Athena Iceberg](https://github.com/aws/aws-sdk-pandas/blob/main/tutorials/039%20-%20Athena%20Iceberg.ipynb) - [040 - EMR Serverless](https://github.com/aws/aws-sdk-pandas/blob/main/tutorials/040%20-%20EMR%20Serverless.ipynb) - [041 - Apache Spark on Amazon Athena](https://github.com/aws/aws-sdk-pandas/blob/main/tutorials/041%20-%20Apache%20Spark%20on%20Amazon%20Athena.ipynb) -- [**API Reference**](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html) - - [Amazon S3](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-s3) - - [AWS Glue Catalog](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#aws-glue-catalog) - - [Amazon Athena](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-athena) - - [Amazon Redshift](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-redshift) - - [PostgreSQL](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#postgresql) - - [MySQL](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#mysql) - - [SQL Server](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#sqlserver) - - [Oracle](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#oracle) - - [Data API Redshift](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#data-api-redshift) - - [Data API RDS](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#data-api-rds) - - [OpenSearch](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#opensearch) - - [AWS Glue Data Quality](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#aws-glue-data-quality) - - [Amazon Neptune](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-neptune) - - [DynamoDB](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#dynamodb) - - [Amazon Timestream](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-timestream) - - [Amazon EMR](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-emr) - - [Amazon CloudWatch Logs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-cloudwatch-logs) - - [Amazon Chime](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-chime) - - [Amazon QuickSight](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#amazon-quicksight) - - [AWS STS](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#aws-sts) - - [AWS Secrets Manager](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#aws-secrets-manager) - - [Global Configurations](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#global-configurations) - - [Distributed - Ray](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html#distributed-ray) +- [**API Reference**](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html) + - [Amazon S3](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-s3) + - [AWS Glue Catalog](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#aws-glue-catalog) + - [Amazon Athena](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-athena) + - [Amazon Redshift](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-redshift) + - [PostgreSQL](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#postgresql) + - [MySQL](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#mysql) + - [SQL Server](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#sqlserver) + - [Oracle](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#oracle) + - [Data API Redshift](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#data-api-redshift) + - [Data API RDS](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#data-api-rds) + - [OpenSearch](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#opensearch) + - [AWS Glue Data Quality](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#aws-glue-data-quality) + - [Amazon Neptune](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-neptune) + - [DynamoDB](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#dynamodb) + - [Amazon Timestream](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-timestream) + - [Amazon EMR](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-emr) + - [Amazon CloudWatch Logs](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-cloudwatch-logs) + - [Amazon Chime](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-chime) + - [Amazon QuickSight](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#amazon-quicksight) + - [AWS STS](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#aws-sts) + - [AWS Secrets Manager](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#aws-secrets-manager) + - [Global Configurations](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#global-configurations) + - [Distributed - Ray](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html#distributed-ray) - [**License**](https://github.com/aws/aws-sdk-pandas/blob/main/LICENSE.txt) - [**Contributing**](https://github.com/aws/aws-sdk-pandas/blob/main/CONTRIBUTING.md) diff --git a/VERSION b/VERSION index 311277513..e06d07afe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.9.2b1 \ No newline at end of file +3.10.0 \ No newline at end of file diff --git a/awswrangler/__metadata__.py b/awswrangler/__metadata__.py index 044030a6d..0cb946908 100644 --- a/awswrangler/__metadata__.py +++ b/awswrangler/__metadata__.py @@ -7,5 +7,5 @@ __title__: str = "awswrangler" __description__: str = "Pandas on AWS." -__version__: str = "3.9.2b1" +__version__: str = "3.10.0" __license__: str = "Apache License 2.0" diff --git a/awswrangler/athena/_read.py b/awswrangler/athena/_read.py index 3c97bc08d..0286ce5de 100644 --- a/awswrangler/athena/_read.py +++ b/awswrangler/athena/_read.py @@ -793,11 +793,11 @@ def read_sql_query( **Related tutorial:** - - `Amazon Athena `_ - - `Athena Cache `_ - - `Global Configurations `_ **There are three approaches available through ctas_approach and unload_approach parameters:** @@ -861,7 +861,7 @@ def read_sql_query( /athena.html#Athena.Client.get_query_execution>`_ . For a practical example check out the - `related tutorial `_! @@ -1137,11 +1137,11 @@ def read_sql_table( **Related tutorial:** - - `Amazon Athena `_ - - `Athena Cache `_ - - `Global Configurations `_ **There are three approaches available through ctas_approach and unload_approach parameters:** @@ -1205,7 +1205,7 @@ def read_sql_table( /athena.html#Athena.Client.get_query_execution>`_ . For a practical example check out the - `related tutorial `_! diff --git a/awswrangler/catalog/_create.py b/awswrangler/catalog/_create.py index 7bb0923d1..e9a566ed6 100644 --- a/awswrangler/catalog/_create.py +++ b/awswrangler/catalog/_create.py @@ -1100,7 +1100,7 @@ def create_csv_table( If True allows schema evolution (new or missing columns), otherwise a exception will be raised. (Only considered if dataset=True and mode in ("append", "overwrite_partitions")) Related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/014%20-%20Schema%20Evolution.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/014%20-%20Schema%20Evolution.html sep String of length 1. Field delimiter for the output file. skip_header_line_count @@ -1280,7 +1280,7 @@ def create_json_table( If True allows schema evolution (new or missing columns), otherwise a exception will be raised. (Only considered if dataset=True and mode in ("append", "overwrite_partitions")) Related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/014%20-%20Schema%20Evolution.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/014%20-%20Schema%20Evolution.html serde_library Specifies the SerDe Serialization library which will be used. You need to provide the Class library name as a string. diff --git a/awswrangler/s3/_read_orc.py b/awswrangler/s3/_read_orc.py index 98519bebe..b7d38c2ff 100644 --- a/awswrangler/s3/_read_orc.py +++ b/awswrangler/s3/_read_orc.py @@ -225,7 +225,7 @@ def read_orc( must return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/023%20-%20Flexible%20Partitions%20Filter.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/023%20-%20Flexible%20Partitions%20Filter.html columns List of columns to read from the file(s). validate_schema @@ -384,7 +384,7 @@ def read_orc_table( must return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/023%20-%20Flexible%20Partitions%20Filter.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/023%20-%20Flexible%20Partitions%20Filter.html columns List of columns to read from the file(s). validate_schema diff --git a/awswrangler/s3/_read_parquet.py b/awswrangler/s3/_read_parquet.py index 1c2d7cb2a..f90c25b69 100644 --- a/awswrangler/s3/_read_parquet.py +++ b/awswrangler/s3/_read_parquet.py @@ -398,7 +398,7 @@ def read_parquet( must return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/023%20-%20Flexible%20Partitions%20Filter.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/023%20-%20Flexible%20Partitions%20Filter.html columns List of columns to read from the file(s). validate_schema @@ -639,7 +639,7 @@ def read_parquet_table( must return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/023%20-%20Flexible%20Partitions%20Filter.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/023%20-%20Flexible%20Partitions%20Filter.html columns List of columns to read from the file(s). validate_schema diff --git a/awswrangler/s3/_read_text.py b/awswrangler/s3/_read_text.py index d8ab7197c..9993adf8d 100644 --- a/awswrangler/s3/_read_text.py +++ b/awswrangler/s3/_read_text.py @@ -236,7 +236,7 @@ def read_csv( This function MUST return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/023%20-%20Flexible%20Partitions%20Filter.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/023%20-%20Flexible%20Partitions%20Filter.html s3_additional_kwargs Forwarded to botocore requests. ray_args @@ -397,7 +397,7 @@ def read_fwf( This function MUST return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/023%20-%20Flexible%20Partitions%20Filter.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/023%20-%20Flexible%20Partitions%20Filter.html s3_additional_kwargs Forwarded to botocore requests. ray_args @@ -565,7 +565,7 @@ def read_json( This function MUST return a bool, True to read the partition or False to ignore it. Ignored if `dataset=False`. E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False`` - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/023%20-%20Flexible%20Partitions%20Filter.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/023%20-%20Flexible%20Partitions%20Filter.html s3_additional_kwargs Forwarded to botocore requests. ray_args diff --git a/awswrangler/s3/_write_orc.py b/awswrangler/s3/_write_orc.py index 37f063a5a..6cb4bdc47 100644 --- a/awswrangler/s3/_write_orc.py +++ b/awswrangler/s3/_write_orc.py @@ -407,7 +407,7 @@ def to_orc( concurrent_partitioning If True will increase the parallelism level during the partitions writing. It will decrease the writing time and increase the memory usage. - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html mode ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True. catalog_versioning @@ -416,7 +416,7 @@ def to_orc( If True allows schema evolution (new or missing columns), otherwise a exception will be raised. True by default. (Only considered if dataset=True and mode in ("append", "overwrite_partitions")) Related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/014%20-%20Schema%20Evolution.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/014%20-%20Schema%20Evolution.html database Glue/Athena catalog: Database name. table diff --git a/awswrangler/s3/_write_parquet.py b/awswrangler/s3/_write_parquet.py index 9203bab9f..8fac36a43 100644 --- a/awswrangler/s3/_write_parquet.py +++ b/awswrangler/s3/_write_parquet.py @@ -435,18 +435,18 @@ def to_parquet( concurrent_partitioning If True will increase the parallelism level during the partitions writing. It will decrease the writing time and increase the memory usage. - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html mode ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True. For details check the related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/004%20-%20Parquet%20Datasets.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/004%20-%20Parquet%20Datasets.html catalog_versioning If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it. schema_evolution If True allows schema evolution (new or missing columns), otherwise a exception will be raised. True by default. (Only considered if dataset=True and mode in ("append", "overwrite_partitions")) Related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/014%20-%20Schema%20Evolution.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/014%20-%20Schema%20Evolution.html database Glue/Athena catalog: Database name. table diff --git a/awswrangler/s3/_write_text.py b/awswrangler/s3/_write_text.py index afb34bb0d..dec033118 100644 --- a/awswrangler/s3/_write_text.py +++ b/awswrangler/s3/_write_text.py @@ -171,18 +171,18 @@ def to_csv( # noqa: PLR0912,PLR0915 concurrent_partitioning If True will increase the parallelism level during the partitions writing. It will decrease the writing time and increase the memory usage. - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html mode ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True. For details check the related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet catalog_versioning If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it. schema_evolution If True allows schema evolution (new or missing columns), otherwise a exception will be raised. (Only considered if dataset=True and mode in ("append", "overwrite_partitions")). False by default. Related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/014%20-%20Schema%20Evolution.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/014%20-%20Schema%20Evolution.html database Glue/Athena catalog: Database name. table @@ -717,18 +717,18 @@ def to_json( # noqa: PLR0912,PLR0915 concurrent_partitioning If True will increase the parallelism level during the partitions writing. It will decrease the writing time and increase the memory usage. - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html mode ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True. For details check the related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet catalog_versioning If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it. schema_evolution If True allows schema evolution (new or missing columns), otherwise a exception will be raised. (Only considered if dataset=True and mode in ("append", "overwrite_partitions")) Related tutorial: - https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/tutorials/014%20-%20Schema%20Evolution.html + https://aws-sdk-pandas.readthedocs.io/en/3.10.0/tutorials/014%20-%20Schema%20Evolution.html database Glue/Athena catalog: Database name. table diff --git a/docs/source/install.rst b/docs/source/install.rst index e7fc7d0b5..7cfcc81f4 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -221,7 +221,7 @@ Go to your Glue PySpark job and create a new *Job parameters* key/value: To install a specific version, set the value for the above Job parameter as follows: -* Value: ``pyarrow==14,pandas==1.5.3,awswrangler==3.9.2b1`` +* Value: ``pyarrow==14,pandas==1.5.3,awswrangler==3.10.0`` `Official Glue PySpark Reference `_ diff --git a/pyproject.toml b/pyproject.toml index 7afb0346b..329195e32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "awswrangler" -version = "3.9.2b1" +version = "3.10.0" description = "Pandas on AWS." authors = ["Amazon Web Services"] license = "Apache License 2.0" diff --git a/test_infra/pyproject.toml b/test_infra/pyproject.toml index 51d004dce..aab98e52e 100644 --- a/test_infra/pyproject.toml +++ b/test_infra/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "awswrangler - test infrastructure" -version = "3.9.2b1" +version = "3.10.0" description = "CDK test infrastructure for AWS SDK for pandas" authors = ["Amazon Web Services"] license = "Apache License 2.0" diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index b27eddc09..dd5af498e 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -2,7 +2,7 @@ def test_metadata(): - assert wr.__version__ == "3.9.2b1" + assert wr.__version__ == "3.10.0" assert wr.__title__ == "awswrangler" assert wr.__description__ == "Pandas on AWS." assert wr.__license__ == "Apache License 2.0" diff --git a/tutorials/001 - Introduction.ipynb b/tutorials/001 - Introduction.ipynb index 2714f969d..0bb088e3c 100644 --- a/tutorials/001 - Introduction.ipynb +++ b/tutorials/001 - Introduction.ipynb @@ -20,7 +20,7 @@ "\n", "Built on top of other open-source projects like [Pandas](https://github.com/pandas-dev/pandas), [Apache Arrow](https://github.com/apache/arrow) and [Boto3](https://github.com/boto/boto3), it offers abstracted functions to execute usual ETL tasks like load/unload data from **Data Lakes**, **Data Warehouses** and **Databases**.\n", "\n", - "Check our [list of functionalities](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html)." + "Check our [list of functionalities](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html)." ] }, { @@ -32,15 +32,15 @@ "\n", "awswrangler runs almost anywhere over Python 3.8, 3.9 and 3.10, so there are several different ways to install it in the desired environment.\n", "\n", - " - [PyPi (pip)](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#pypi-pip)\n", - " - [Conda](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#conda)\n", - " - [AWS Lambda Layer](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#aws-lambda-layer)\n", - " - [AWS Glue Python Shell Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#aws-glue-python-shell-jobs)\n", - " - [AWS Glue PySpark Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#aws-glue-pyspark-jobs)\n", - " - [Amazon SageMaker Notebook](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#amazon-sagemaker-notebook)\n", - " - [Amazon SageMaker Notebook Lifecycle](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#amazon-sagemaker-notebook-lifecycle)\n", - " - [EMR Cluster](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#emr-cluster)\n", - " - [From source](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/install.html#from-source)\n", + " - [PyPi (pip)](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#pypi-pip)\n", + " - [Conda](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#conda)\n", + " - [AWS Lambda Layer](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#aws-lambda-layer)\n", + " - [AWS Glue Python Shell Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#aws-glue-python-shell-jobs)\n", + " - [AWS Glue PySpark Jobs](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#aws-glue-pyspark-jobs)\n", + " - [Amazon SageMaker Notebook](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#amazon-sagemaker-notebook)\n", + " - [Amazon SageMaker Notebook Lifecycle](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#amazon-sagemaker-notebook-lifecycle)\n", + " - [EMR Cluster](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#emr-cluster)\n", + " - [From source](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/install.html#from-source)\n", "\n", "Some good practices for most of the above methods are:\n", " - Use new and individual Virtual Environments for each project ([venv](https://docs.python.org/3/library/venv.html))\n", diff --git a/tutorials/007 - Redshift, MySQL, PostgreSQL, SQL Server, Oracle.ipynb b/tutorials/007 - Redshift, MySQL, PostgreSQL, SQL Server, Oracle.ipynb index 8e6c16e67..0dc6bcc72 100644 --- a/tutorials/007 - Redshift, MySQL, PostgreSQL, SQL Server, Oracle.ipynb +++ b/tutorials/007 - Redshift, MySQL, PostgreSQL, SQL Server, Oracle.ipynb @@ -10,16 +10,16 @@ "\n", "[awswrangler](https://github.com/aws/aws-sdk-pandas)'s Redshift, MySQL and PostgreSQL have two basic functions in common that try to follow Pandas conventions, but add more data type consistency.\n", "\n", - "- [wr.redshift.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.redshift.to_sql.html)\n", - "- [wr.redshift.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.redshift.read_sql_query.html)\n", - "- [wr.mysql.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.mysql.to_sql.html)\n", - "- [wr.mysql.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.mysql.read_sql_query.html)\n", - "- [wr.postgresql.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.postgresql.to_sql.html)\n", - "- [wr.postgresql.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.postgresql.read_sql_query.html)\n", - "- [wr.sqlserver.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.sqlserver.to_sql.html)\n", - "- [wr.sqlserver.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.sqlserver.read_sql_query.html)\n", - "- [wr.oracle.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.oracle.to_sql.html)\n", - "- [wr.oracle.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.oracle.read_sql_query.html)" + "- [wr.redshift.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.redshift.to_sql.html)\n", + "- [wr.redshift.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.redshift.read_sql_query.html)\n", + "- [wr.mysql.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.mysql.to_sql.html)\n", + "- [wr.mysql.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.mysql.read_sql_query.html)\n", + "- [wr.postgresql.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.postgresql.to_sql.html)\n", + "- [wr.postgresql.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.postgresql.read_sql_query.html)\n", + "- [wr.sqlserver.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.sqlserver.to_sql.html)\n", + "- [wr.sqlserver.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.sqlserver.read_sql_query.html)\n", + "- [wr.oracle.to_sql()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.oracle.to_sql.html)\n", + "- [wr.oracle.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.oracle.read_sql_query.html)" ] }, { @@ -51,11 +51,11 @@ "source": [ "## Connect using the Glue Catalog Connections\n", "\n", - "- [wr.redshift.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.redshift.connect.html)\n", - "- [wr.mysql.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.mysql.connect.html)\n", - "- [wr.postgresql.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.postgresql.connect.html)\n", - "- [wr.sqlserver.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.sqlserver.connect.html)\n", - "- [wr.oracle.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.oracle.connect.html)" + "- [wr.redshift.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.redshift.connect.html)\n", + "- [wr.mysql.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.mysql.connect.html)\n", + "- [wr.postgresql.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.postgresql.connect.html)\n", + "- [wr.sqlserver.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.sqlserver.connect.html)\n", + "- [wr.oracle.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.oracle.connect.html)" ] }, { diff --git a/tutorials/014 - Schema Evolution.ipynb b/tutorials/014 - Schema Evolution.ipynb index ba3fbe91c..e810a2c51 100644 --- a/tutorials/014 - Schema Evolution.ipynb +++ b/tutorials/014 - Schema Evolution.ipynb @@ -10,9 +10,9 @@ "\n", "awswrangler supports new **columns** on Parquet and CSV datasets through:\n", "\n", - "- [wr.s3.to_parquet()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet)\n", - "- [wr.s3.store_parquet_metadata()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.s3.store_parquet_metadata.html#awswrangler.s3.store_parquet_metadata) i.e. \"Crawler\"\n", - "- [wr.s3.to_csv()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.s3.to_csv.html#awswrangler.s3.to_csv)" + "- [wr.s3.to_parquet()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet)\n", + "- [wr.s3.store_parquet_metadata()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.s3.store_parquet_metadata.html#awswrangler.s3.store_parquet_metadata) i.e. \"Crawler\"\n", + "- [wr.s3.to_csv()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.s3.to_csv.html#awswrangler.s3.to_csv)" ] }, { diff --git a/tutorials/021 - Global Configurations.ipynb b/tutorials/021 - Global Configurations.ipynb index 6dff53d7e..39ae714c5 100644 --- a/tutorials/021 - Global Configurations.ipynb +++ b/tutorials/021 - Global Configurations.ipynb @@ -13,7 +13,7 @@ "- **Environment variables**\n", "- **wr.config**\n", "\n", - "*P.S. Check the [function API doc](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html) to see if your function has some argument that can be configured through Global configurations.*\n", + "*P.S. Check the [function API doc](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html) to see if your function has some argument that can be configured through Global configurations.*\n", "\n", "*P.P.S. One exception to the above mentioned rules is the `botocore_config` property. It cannot be set through environment variables\n", "but only via `wr.config`. It will be used as the `botocore.config.Config` for all underlying `boto3` calls.\n", diff --git a/tutorials/022 - Writing Partitions Concurrently.ipynb b/tutorials/022 - Writing Partitions Concurrently.ipynb index 42aeb4a8e..d7073dc85 100644 --- a/tutorials/022 - Writing Partitions Concurrently.ipynb +++ b/tutorials/022 - Writing Partitions Concurrently.ipynb @@ -13,7 +13,7 @@ " If True will increase the parallelism level during the partitions writing. It will decrease the\n", " writing time and increase memory usage.\n", "\n", - "*P.S. Check the [function API doc](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html) to see it has some argument that can be configured through Global configurations.*" + "*P.S. Check the [function API doc](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html) to see it has some argument that can be configured through Global configurations.*" ] }, { diff --git a/tutorials/023 - Flexible Partitions Filter.ipynb b/tutorials/023 - Flexible Partitions Filter.ipynb index e72e922c5..3838d08fa 100644 --- a/tutorials/023 - Flexible Partitions Filter.ipynb +++ b/tutorials/023 - Flexible Partitions Filter.ipynb @@ -16,7 +16,7 @@ " - Ignored if `dataset=False`.\n", " \n", "\n", - "*P.S. Check the [function API doc](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/api.html) to see it has some argument that can be configured through Global configurations.*" + "*P.S. Check the [function API doc](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/api.html) to see it has some argument that can be configured through Global configurations.*" ] }, { diff --git a/tutorials/030 - Data Api.ipynb b/tutorials/030 - Data Api.ipynb index 960fbde8b..c3e0a17cf 100644 --- a/tutorials/030 - Data Api.ipynb +++ b/tutorials/030 - Data Api.ipynb @@ -26,8 +26,8 @@ "metadata": {}, "source": [ "## Connect to the cluster\n", - "- [wr.data_api.redshift.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.data_api.redshift.connect.html)\n", - "- [wr.data_api.rds.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.data_api.rds.connect.html)" + "- [wr.data_api.redshift.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.data_api.redshift.connect.html)\n", + "- [wr.data_api.rds.connect()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.data_api.rds.connect.html)" ] }, { @@ -60,8 +60,8 @@ "metadata": {}, "source": [ "## Read from database\n", - "- [wr.data_api.redshift.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.data_api.redshift.read_sql_query.html)\n", - "- [wr.data_api.rds.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.9.2b1/stubs/awswrangler.data_api.rds.read_sql_query.html)" + "- [wr.data_api.redshift.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.data_api.redshift.read_sql_query.html)\n", + "- [wr.data_api.rds.read_sql_query()](https://aws-sdk-pandas.readthedocs.io/en/3.10.0/stubs/awswrangler.data_api.rds.read_sql_query.html)" ] }, {